[ May 30 04:07:46 Enabled. ] [ May 30 04:07:46 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ May 30 04:07:46 Method "start" exited with status 0. ] [ May 30 04:07:46 Rereading configuration. ] [ May 30 04:07:46 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:46.386432983Z","hostname":"sock","pid":1562,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:46.386601156Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:46.439676159Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:46.439721069Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:46.555569027Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:46.564888753Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:46.572809184Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.175731454Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.313639893Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.313681574Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.313789966Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.313892908Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.313909258Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.313930019Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.317094625Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:47.317100645Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.317112005Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.317115535Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.317130186Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.317194397Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.317208277Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.317223177Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.317250818Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.328199392Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.328230223Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"75663f09-f516-40fc-852b-77e4a30319f5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.425645663Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.628474426Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.628514477Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.628522427Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.628528327Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.628541307Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.628545798Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.628563298Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.637152011Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.637176931Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"804b9788-bd41-4d87-a91b-e568478fa644 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.729129554Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930643494Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930693615Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930720056Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930732276Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930745586Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930751186Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930755936Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930760776Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930771407Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930776697Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930781197Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930808407Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930842758Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930854068Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930858008Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930879849Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930884289Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930889769Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930893089Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930896519Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930910429Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930915389Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930919519Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930925479Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930929049Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930933919Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.930937699Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.93096058Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.93096505Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.93096908Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.93097241Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.93097716Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.9309809Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.93098496Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931001491Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931006351Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931010721Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931016061Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931039101Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931043001Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931047812Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931051541Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931055721Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931062052Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931077952Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931081642Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931088052Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931092662Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931096102Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931099352Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931124403Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931129163Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931133413Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931137363Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931140823Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931151973Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931155563Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931182794Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931187674Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931191104Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931194734Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931233265Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931250415Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931262995Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931280785Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931285136Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931289026Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931315506Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.931322926Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.94167443Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.941700181Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999926328s","next_cleanup":"Instant { tv_sec: 1264, tv_nsec: 30570189 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.941764712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.941770502Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3645"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.941811103Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.941822773Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.944752085Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.944810456Z","hostname":"sock","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.94502948Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1004"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.94503664Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945084201Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945116771Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945126291Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945131222Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1064"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945154402Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945168242Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945176952Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945194853Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1105"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945198423Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945201913Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945208083Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945222073Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945529919Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945675371Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945700502Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1138"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945748913Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945758473Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.945904875Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946036228Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946078139Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: dd908086-f372-434c-b66b-6e5770f567d8, rack_id: 4559c8be-8414-459c-b1cf-c6595c790cca, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946205521Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:33291","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: dd908086-f372-434c-b66b-6e5770f567d8, rack_id: 4559c8be-8414-459c-b1cf-c6595c790cca, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946342443Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946351583Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946378404Z","hostname":"sock","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946391374Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946397544Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946401394Z","hostname":"sock","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946405744Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946429295Z","hostname":"sock","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946436375Z","hostname":"sock","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946443525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.946448935Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.949148193Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.956357761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.965996022Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.976084841Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.985180102Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:47.99462138Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.003353125Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.012150682Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.021264664Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.033662464Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.033730105Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.033738665Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.033747715Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.033755525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.033764176Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.056805675Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.075374905Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.085430963Z","hostname":"sock","pid":1562,"sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:353"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.085463814Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.085481184Z","hostname":"sock","pid":1562,"sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:365"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.085530895Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 145.333895ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.08858401Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d91df1e7-b013-4c8a-aa80-2400dc2b49ec\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:07:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d91df1e7-b013-4c8a-aa80-2400dc2b49ec\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_75663f09-f516-40fc-852b-77e4a30319f5/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.098791741Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_75663f09-f516-40fc-852b-77e4a30319f5/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.1224033Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_75663f09-f516-40fc-852b-77e4a30319f5/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.122443991Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.127729785Z","hostname":"sock","pid":1562,"sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:380"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.127739225Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_75663f09-f516-40fc-852b-77e4a30319f5/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.127744585Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.198123155Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.234223137Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 144.065623ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.23498308Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"00896ef7-f5c2-4989-8d69-1671b48f2860\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:07:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"00896ef7-f5c2-4989-8d69-1671b48f2860\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.381071845Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 204.58175ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.381656776Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"74c6382c-e269-418b-9854-19137bc134d0\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:07:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"74c6382c-e269-418b-9854-19137bc134d0\" }"} {"msg":"Mounting oxi_75663f09-f516-40fc-852b-77e4a30319f5/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.417137236Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.429396364Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.429449165Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.429457195Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.429473765Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.587546853Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 169.32606ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.587954241Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11e5fc82-eb25-45f8-8983-68b743edba89\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:07:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"11e5fc82-eb25-45f8-8983-68b743edba89\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.759075721Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 209.916819ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.759941476Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f5c3e036-c184-4d28-acf3-618e817a1a30\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:07:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f5c3e036-c184-4d28-acf3-618e817a1a30\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.971060326Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 324.662895ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:48.971793839Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3a87dbe6-7d3b-4c49-826e-fadebc5b8c37\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:07:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3a87dbe6-7d3b-4c49-826e-fadebc5b8c37\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.153065399Z","hostname":"sock","pid":1562,"sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.15366103Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.15367993Z","hostname":"sock","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.153737921Z","hostname":"sock","pid":1562,"file":"oximeter/producer/src/lib.rs:276","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:276","interval":"30s","address":"[fd00:1122:3344:101::1]:39972","producer_id":"dd908086-f372-434c-b66b-6e5770f567d8"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:49.15988935Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"361.926412ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.200697365Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.200739096Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.200835888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.200846098Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.201649682Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.201683993Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.201715843Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.201732433Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.202032059Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.202207292Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.202230052Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [V4(Ipv4Net { addr: 10.151.2.174, width: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.202254253Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3645"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.202276483Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:33291","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.396089466Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.396145807Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.396152477Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.39629184Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.396375571Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.396390962Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.396396492Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.396408232Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:49.400520565Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.428048104Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64552"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.428215277Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.428293758Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"08c74728-9637-4444-85f5-450f11cfed42","remote_addr":"[fd00:1122:3344:101::1]:64552","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.457057819Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.457162731Z","hostname":"sock","pid":1562,"uri":"/sled-role","method":"GET","req_id":"e6c3b07d-bb59-444c-97fc-04bcf74e25d2","remote_addr":"[fd00:1122:3344:101::1]:52111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":58,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.457839363Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.457855003Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.45825877Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.458476534Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 70e55cf7-953e-4643-9a59-9e2958d5bc87, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool) }, OmicronPhysicalDiskConfig { id: a5ae2404-80a9-4819-ba5e-6728732e3598, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 1793c88e-7478-4741-8254-860fae35146b (zpool) }, OmicronPhysicalDiskConfig { id: ef3821e0-ed58-4b96-afa2-47b3bd30eaef, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool) }, OmicronPhysicalDiskConfig { id: b9d5e503-3f68-4e2f-a22a-c4f8a87f5fdb, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool) }, OmicronPhysicalDiskConfig { id: 35cc8755-1c2b-423b-bedb-b6aeab45dbbb, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool) }, OmicronPhysicalDiskConfig { id: d10cffb8-7892-457e-a907-a90de19f4f0d, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: a168fba6-15db-4743-814f-b76d3daf999f (zpool) }, OmicronPhysicalDiskConfig { id: 27df9c70-ad52-401d-a29d-6e10f7b3836a, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool) }, OmicronPhysicalDiskConfig { id: d40d83f5-e344-4e52-bad7-855a94aa96e3, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool) }, OmicronPhysicalDiskConfig { id: daa27901-b4fa-454a-be6d-f824b6019b4c, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.489699639Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.489812601Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64395"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 70e55cf7-953e-4643-9a59-9e2958d5bc87, pool_id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: a5ae2404-80a9-4819-ba5e-6728732e3598, pool_id: 1793c88e-7478-4741-8254-860fae35146b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: ef3821e0-ed58-4b96-afa2-47b3bd30eaef, pool_id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: b9d5e503-3f68-4e2f-a22a-c4f8a87f5fdb, pool_id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 35cc8755-1c2b-423b-bedb-b6aeab45dbbb, pool_id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: d10cffb8-7892-457e-a907-a90de19f4f0d, pool_id: a168fba6-15db-4743-814f-b76d3daf999f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 27df9c70-ad52-401d-a29d-6e10f7b3836a, pool_id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: d40d83f5-e344-4e52-bad7-855a94aa96e3, pool_id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: daa27901-b4fa-454a-be6d-f824b6019b4c, pool_id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.490023335Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.490029085Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.490042855Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.490047735Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.490062935Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.490067756Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.490080436Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.490084956Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:49.606414042Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"352.708911ms"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.612699524Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:49.612714214Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:50.011683152Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"523.904099ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:07:50.279953127Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:50.684183568Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.93510184s"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:50.693948092Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:50.889051638Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:50.88918264Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:50.88920452Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:50.88921283Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:50.889223411Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:50.889386953Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:50.889395514Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:51.305835561Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:51.740042544Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:07:51.740280259Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:51.960370908Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 47.730227ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:52.167762922Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Automatically destroying dataset: oxp_3022fa0e-6566-4854-a1a9-1e0c5908d5b0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:52.173473634Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 106.656691ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:52.77276553Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:52.774574752Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.138147591s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:07:52.774654023Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 158.827502ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:53.437237663Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:53.438461645Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 493.231977ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:54.035361718Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:07:54.037312283Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.070324886s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:55.102699288Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:07:55.104136524Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_3022fa0e-6566-4854-a1a9-1e0c5908d5b0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:55.114375056Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:55.309526243Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:55.309585973Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:55.309592874Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:55.309598744Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:55.309602664Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:55.509751009Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:55.50977755Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"1793c88e-7478-4741-8254-860fae35146b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:56.57092825Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:56.798544294Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:56.798587264Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:56.798593645Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:56.798615965Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:56.798623235Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:56.798766468Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:56.798787418Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:57.669599167Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:07:57.901815082Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:58.121425303Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:58.340342922Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"11.81069839s"} {"msg":"Automatically destroying dataset: oxp_1793c88e-7478-4741-8254-860fae35146b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:58.578118276Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:07:59.230456314Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:07:59.230853661Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.432131317Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_1793c88e-7478-4741-8254-860fae35146b","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.540898049Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.54885507Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.548897981Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.548908471Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.548919691Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.548928011Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.55839736Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.55842474Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"f61d361f-e4c0-48df-9191-ff705041ee76 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.653385407Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.661380249Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.66141869Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.66142585Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.66143228Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.66146165Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.661580002Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.661587903Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.695889602Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f61d361f-e4c0-48df-9191-ff705041ee76/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.704195279Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_f61d361f-e4c0-48df-9191-ff705041ee76","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.826236067Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.834118578Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.834157018Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.834164338Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.834171208Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.834175919Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.843723948Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.843751529Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.935282185Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.943001882Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.943044133Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.943050493Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.943085193Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.943096043Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.943253886Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.943277817Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.976021888Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:07:59.9834573Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.106028577Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.113741755Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.113771395Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.113789586Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.113806256Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.113811976Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.12361395Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.123648911Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.219468933Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.227264811Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.227315712Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.227326532Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.227353813Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.227367813Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.227519496Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.227528386Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:08:00.232530935Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.262890664Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_5d12239f-a80e-4f75-83d4-ebef24b6fea1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.270650112Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_5d12239f-a80e-4f75-83d4-ebef24b6fea1","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.392264742Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.400174273Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.400211234Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.400217794Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.400224704Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.400229674Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.409698422Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.409720532Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"a168fba6-15db-4743-814f-b76d3daf999f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.501699777Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.509651578Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.509693519Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.509699819Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.50977357Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.50978653Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.509873922Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.509880822Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.545019986Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a168fba6-15db-4743-814f-b76d3daf999f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.554420103Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_a168fba6-15db-4743-814f-b76d3daf999f","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.676286868Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.684071616Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.684122107Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.684130657Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.684140287Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.684146898Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.693560295Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.693593045Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.788475321Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.79629002Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.796329111Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.796353901Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.796398882Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.796432042Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.796639306Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.796648306Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:00.80079859Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.834219324Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.84300563Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.918738245Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2331"} {"msg":"Looking for unencrypted datasets in oxp_2efcf3a6-ef7f-48c2-8713-0dd19fe385d1","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.968422607Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.975628585Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.975659645Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.975665476Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.975670636Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.975676626Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.984983751Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:00.985011031Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.079399938Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.086853921Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.086883301Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.086900612Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.086951362Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.086964763Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.087112585Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.087121256Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.121997965Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_dc70dd36-578e-4d7e-9ca2-11c36573efc3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.13018347Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:08:01.234484103Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_dc70dd36-578e-4d7e-9ca2-11c36573efc3","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.254844735Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.262770996Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.262806726Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.262813737Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.262821647Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.262841917Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.271969359Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.27199949Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.367230681Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.374729175Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.374766505Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.374773575Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.374786316Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.374811586Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.374976629Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.374989259Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.410563481Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8f547a7a-6177-4ca0-82ce-6646cf05d8a6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.418405081Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_8f547a7a-6177-4ca0-82ce-6646cf05d8a6","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.542284771Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.550250603Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.550287484Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.550350374Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.550522258Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 75663f09-f516-40fc-852b-77e4a30319f5 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 804b9788-bd41-4d87-a91b-e568478fa644 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.550578418Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.550586059Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.550703861Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.551030627Z","hostname":"sock","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"6b486cd9-72dd-4ba2-a5a5-7acbb35f629c","remote_addr":"[fd00:1122:3344:101::1]:64395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12061150,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.581425957Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.581573299Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53600"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.581888525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.581898165Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.632029756Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.632074116Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.681783469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.68182099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.732293517Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.732340898Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.732346538Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.732384138Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.776046184Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.776087415Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.776093655Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.826181334Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.829624486Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.833162309Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.845906845Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.850279693Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:01.854206732Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-30T04:08:02.236546584Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/dd908086-f372-434c-b66b-6e5770f567d8): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:02.8014697Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Profile for oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:03.364065304Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:03.364120805Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:03.844169182Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:04.322081952Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:04.322205874Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:04.322273325Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:04.326303317Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:04.326325807Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:04.330283497Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd\") failed: Property not found. retry in 68.889482ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.453326353Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 40.069723ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.460050393Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982\") failed: Property not found. retry in 25.333895ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.466530488Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982\") failed: Property not found. retry in 56.673419ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.499155647Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 52.554987ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.507071288Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd\") failed: Property not found. retry in 113.961931ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.57643504Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 170.528916ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.649075551Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982\") failed: Property not found. retry in 278.59887ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.694778903Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd\") failed: Property not found. retry in 126.61523ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.880510012Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 234.11341ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:04.907179886Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982\") failed: Property not found. retry in 567.830602ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:05.019351008Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd\") failed: Property not found. retry in 307.225684ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:05.070449956Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 447.532451ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:05.185396407Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd\") failed: Property not found. retry in 825.876859ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:05.38929781Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982\") failed: Property not found. retry in 1.182060878s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:05.588888765Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 770.423937ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:05.643274032Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd\") failed: Property not found. retry in 1.354237491s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:06.225119567Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 1.172663259s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:06.423920168Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982\") failed: Property not found. retry in 592.901854ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:06.780572784Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:07.004476031Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2348"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:07.004503132Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982\") failed: Property not found. retry in 1.291282407s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:07.382326824Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe8e:5929","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:07.557644478Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2355"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:07.563139286Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2416"} {"msg":"wait for service Some(\"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd\") failed: Property not found. retry in 1.023072949s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:07.587294775Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 515.717398ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:07.603779488Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:07.683165848Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2601"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:07.920442663Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2765"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:08.106854314Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2480"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 527.851008ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:08.128825374Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:08.396085782Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2515"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:08.594232362Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2815"} {"msg":"wait for service Some(\"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd\") failed: Property not found. retry in 1.283671878s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:08.619682474Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063\") failed: Property not found. retry in 1.342999536s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:08.665952536Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982\") failed: Property not found. retry in 1.30904213s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:08.682889787Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:08.768331985Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2842"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:08.987566799Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2709"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:09.162767321Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2509"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:09.173006883Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3901","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:09.173046684Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3917"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:09.173056954Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:09.723255248Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3925"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:09.729605931Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3981"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:09.876117214Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.000591734Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4024","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.028892577Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4071"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.032079194Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_internal_dns_a5f19f92-dd9d-4d5d-b88b-df21ddc95063"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.032145915Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_internal_dns_5269a058-580b-4ad2-a686-a68f8b8e0982"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.032151725Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_internal_dns_448a7523-abaa-4f37-8ea0-88fd1a15dcfd"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.032188326Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.032536072Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.032834757Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"de22cb3f-3ee5-4521-a2be-6e336f16c17e","remote_addr":"[fd00:1122:3344:101::1]:53600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8451175,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.034586448Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.163391566Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4098"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.30824413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.339094088Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.370863532Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.370952063Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.37131497Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.37132908Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449693852Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449759403Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449767794Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449775224Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449781304Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449788004Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449794214Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449800844Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449808124Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449814024Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449835485Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449843135Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449850205Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449856425Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449862335Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449867245Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449871405Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449875926Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449880286Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449884806Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449888986Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449893636Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.449902686Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to write DNS configuration (will retry in 89.397296ms)","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:10.639364252Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:569","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.748402009Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.795875972Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.844243701Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.844256322Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.87567537Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.877716906Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.877725796Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.90721172Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.907355023Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42994"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.907534466Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.907544046Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.92914842Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.929201131Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.948997282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.949042833Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.966475853Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.966558994Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.966738858Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.96687223Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:10.978751111Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.011451761Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.012362708Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.012368028Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.056743476Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 169, 65]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.056784237Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.057356277Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.057366437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.057678563Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.059461974Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.083135075Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.101316578Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.718620344Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:11.718705225Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 57.32953ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:11.852854628Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 110.890964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:11.931048978Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 121.337388ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:12.064595389Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 407.872749ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:12.196251518Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 642.716246ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:12.612870719Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 862.086763ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:13.26646251Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:13.67970224Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36630"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:13.679838253Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"8d3c54ff-aec4-4995-b70f-8e3cf20722e4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36630","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":65,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 506.087932ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:14.138233036Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 706.307785ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:14.652645944Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 648.616462ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:15.367455212Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d\") failed: Property not found. retry in 643.845596ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:16.026807244Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.691406Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.691475052Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.691784727Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.692059722Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"961d094e-cc5f-450c-8279-ff20ed752686","remote_addr":"[fd00:1122:3344:101::1]:42994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":5784648,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.693907185Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.722814478Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.722957441Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42525"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.733660901Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.733791993Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"00c1815d-2920-4e2e-9eb6-096243c735cd","remote_addr":"[fd00:1122:3344:101::1]:42525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":10706,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:16.736184326Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.819364393Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:16.819455425Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59146"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:18.82826419Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:18.828330191Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"8689eb65-ad7c-4f56-8277-9283e14413ab","remote_addr":"[fd00:1122:3344:101::1]:59146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":2008777,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:18.830620051Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:18.976655635Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:18.976787348Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51222"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:18.985418141Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:18.985524793Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"eb4e4c15-bc27-4bbc-a29e-3948633b284c","remote_addr":"[fd00:1122:3344:101::1]:51222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8573,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:18.98708116Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:19.099861454Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:19.100009886Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61120"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_37aef7a7-53c8-4b7f-aab3-5292349f3f5d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.107665379Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.107804101Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"5d138095-8a81-4a3f-8353-c1691c55d625","remote_addr":"[fd00:1122:3344:101::1]:61120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007689,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:23.110811565Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.255269171Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.255405803Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:65195"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.264645027Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"46fa1e6d-a9c0-4d92-8bce-9e9e1a9fa614","remote_addr":"[fd00:1122:3344:101::1]:65195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9081,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.267750682Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:23.267763333Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.590668979Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.590785911Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61889"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.598864675Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"1442ad5d-77be-447f-8f53-8bcbd9de3313","remote_addr":"[fd00:1122:3344:101::1]:61889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7940,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.600645706Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:23.600654106Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.953342782Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.953439183Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39470"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.962070037Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"75f083e7-3ae1-49a5-a5f6-21c7c77139e5","remote_addr":"[fd00:1122:3344:101::1]:39470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8490,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:23.963620684Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:23.963629024Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:24.401629565Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:24.401731767Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54670"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:24.409596716Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"b42be9f4-2934-47a8-a3ea-f030bcf6f22b","remote_addr":"[fd00:1122:3344:101::1]:54670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7781,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:24.412794913Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:24.412804003Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:24.86632001Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:24.866443862Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:24.875076945Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"a31d4ca4-7c92-4f1d-9b33-0d14a5a87615","remote_addr":"[fd00:1122:3344:101::1]:49939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8589,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:24.878815202Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:24.878825712Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:25.435636133Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:25.435734555Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59407"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:25.443879639Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"75a17a89-4ca3-4ed6-a5d7-8f1fe92b7120","remote_addr":"[fd00:1122:3344:101::1]:59407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8013,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:25.445898845Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:25.445907565Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:26.220663718Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:26.22078294Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48809"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:26.229423324Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"8e090a5b-2e2c-49b5-8b7b-1085a7b2d9ed","remote_addr":"[fd00:1122:3344:101::1]:48809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8521,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:26.231536081Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:26.231543151Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:26.632170568Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:26.632318911Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55273"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:26.641000105Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"f5836997-8442-4fcb-88cc-4ad555cd62c7","remote_addr":"[fd00:1122:3344:101::1]:55273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8617,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:26.644731931Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:26.644740511Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:27.504366972Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:27.504479974Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:58537"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:27.512505526Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"6eb85e3e-a178-4ae6-bf48-eff64ad174da","remote_addr":"[fd00:1122:3344:101::1]:58537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7886,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:27.514311049Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:27.514319789Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.278180031Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.278285493Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48210"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.299068822Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"31cec257-e59d-4b6f-a100-840859be7c6a","remote_addr":"[fd00:1122:3344:101::1]:48210","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":20621,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2310930948, ip_addr: 137.190.2.4, stratum: 3, ref_time: 1717042106.539167, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.300457967Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.300488068Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1217"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.329581615Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.329701997Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64117"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.329859829Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.32987039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.348768125Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.348801026Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.36648502Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.366528971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.384006431Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.384056712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.436016365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.436062736Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.483703542Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.483743823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.533520587Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.533556308Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.583956293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.583992584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.634752966Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.634807687Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.635048061Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.635192673Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.717394594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.717434245Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.717440685Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.717446425Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.717450475Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.805733133Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.80951969Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.813789116Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.818180164Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.822609603Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.826860168Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.830951111Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.835097275Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.839193957Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:28.844296808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:30.907584124Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"204.901214ms"} {"msg":"Profile for oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.53748712Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.537648503Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.537657604Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.543155511Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.5779678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.578153833Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.578162113Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.583458567Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.591975638Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:32.5920781Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 50.030945ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:32.687009596Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 64.481584ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:32.69396853Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 65.707614ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:32.721048911Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 30.066721ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:32.72776284Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 68.180386ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:32.73392788Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 123.864902ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:32.745904493Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 144.564436ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:33.692746174Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 140.551119ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:33.754958579Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 113.528084ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:34.340644604Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 150.288707ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:34.650398107Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 172.844279ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:34.802104452Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 98.192434ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:34.810328088Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 236.854594ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:34.81720511Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 155.010151ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:34.824612982Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 320.090086ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:34.831671217Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 276.448763ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.071407626Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 466.477567ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.187100282Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 230.382939ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.334131554Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 599.918237ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.578524406Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 422.714506ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.691321759Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 662.28657ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.702089731Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 253.577612ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.712586217Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 442.006455ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.721011197Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 662.35189ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:35.976510655Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 990.641127ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:36.124275421Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 789.11932ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:36.175019313Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 1.085255804s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:36.188788797Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 1.030959505s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:36.375512155Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 623.685334ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:36.649577884Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 613.07901ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:36.972828806Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 1.470519123s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:37.125203533Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 1.344691308s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:37.283110569Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 563.743037ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:37.291436726Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 1.156537442s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:37.415196325Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 883.827054ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:37.595806184Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 773.660105ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:37.8646666Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 1.130102454s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:38.489993879Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 1.348001555s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:38.582065685Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 1.479819742s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:38.606830365Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 980.511603ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:38.638191812Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 656.892098ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:38.647316854Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 1.071025564s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:39.316569914Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 808.285516ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:39.629596305Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 1.367039343s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:39.637931033Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 1.17396571s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:39.93777215Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 1.391676271s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:40.09485447Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 1.471077383s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:40.396083212Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 686.023985ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:40.446013859Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 733.937073ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:41.013651984Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 889.666371ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:41.122238893Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 1.292802015s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:41.141658358Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035\") failed: Property not found. retry in 1.058621215s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:41.496229127Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 1.338391041s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:41.756310767Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0\") failed: Property not found. retry in 931.392505ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:41.875625297Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690\") failed: Property not found. retry in 581.282284ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:42.022568657Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456\") failed: Property not found. retry in 863.306476ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:42.445846237Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 634.08709ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:43.104231834Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 581.150561ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:43.749654179Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 736.439139ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:44.346844879Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:44.450203616Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:44.450383829Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"67820c20-4e1f-4885-8da7-2537e4b2c8ad","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43177","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 625.592505ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:45.095157503Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 747.393684ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:46.185362101Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:46.265667868Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"675.280773ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 1.195558181s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:46.94463774Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 676.809094ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:48.149983084Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 1.227757339s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:48.836801206Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 1.346392462s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:50.075073124Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b\") failed: Property not found. retry in 1.22305523s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:51.43193472Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.677010109Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.677130161Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_72c3976d-68e9-463a-be70-3de38abfe035"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.677140391Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_9112ad3c-6c15-4601-bf24-48148ee77690"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.677152001Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_a84ccf3e-ca81-499f-b526-5c900c243456"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.677164112Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_cockroachdb_fef621b3-c922-4648-a07b-e45ce7dbe3e0"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.677175132Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.678443204Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.721524209Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5f36bab8-1467-41ed-bd47-85c535d1b2ce","remote_addr":"[fd00:1122:3344:101::1]:64117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":23602692,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.754540766Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51068"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.789548528Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"582d0c1d-1cd3-4d2b-a1b3-248378dcfc03","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":34856,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:52.789764872Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"92.830849ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"582d0c1d-1cd3-4d2b-a1b3-248378dcfc03\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"582d0c1d-1cd3-4d2b-a1b3-248378dcfc03\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:52.911277901Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"8d743e38-cbb4-4d8b-ac86-560c7ce7b65b","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27357,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:52.911412303Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"104.159373ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8d743e38-cbb4-4d8b-ac86-560c7ce7b65b\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8d743e38-cbb4-4d8b-ac86-560c7ce7b65b\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:53.042612384Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"6eb262a5-068f-4841-a1e9-2267fcb2d117","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26056,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:53.042742526Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"72.082994ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6eb262a5-068f-4841-a1e9-2267fcb2d117\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6eb262a5-068f-4841-a1e9-2267fcb2d117\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:53.141901858Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"da4a31f4-ef96-4fce-a7d2-e8e8c0d25ab5","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25970,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:53.142068161Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"110.414703ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da4a31f4-ef96-4fce-a7d2-e8e8c0d25ab5\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"da4a31f4-ef96-4fce-a7d2-e8e8c0d25ab5\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:53.279223668Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"ba3aaab6-d87c-4a41-8964-9ffc87f94b4d","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":25944,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:53.27935172Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"219.367422ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ba3aaab6-d87c-4a41-8964-9ffc87f94b4d\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ba3aaab6-d87c-4a41-8964-9ffc87f94b4d\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:53.526131314Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"f89403a7-69d4-4fa5-8581-4e22eb510103","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26545,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:53.526273037Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"237.397981ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f89403a7-69d4-4fa5-8581-4e22eb510103\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f89403a7-69d4-4fa5-8581-4e22eb510103\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:53.790576312Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"71671a25-04b9-4456-9281-b2a33a92788d","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26083,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:53.790731964Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"317.164001ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"71671a25-04b9-4456-9281-b2a33a92788d\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"71671a25-04b9-4456-9281-b2a33a92788d\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:54.135655692Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"ff892501-18ee-446f-9fdc-db2917fd14ae","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":26737,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:54.135819415Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"185.100816ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ff892501-18ee-446f-9fdc-db2917fd14ae\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ff892501-18ee-446f-9fdc-db2917fd14ae\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:54.351328814Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"9a6d0f3d-eab1-49af-9588-6f5fdcf1e7f8","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":29260,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:54.351493877Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"406.050653ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9a6d0f3d-eab1-49af-9588-6f5fdcf1e7f8\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9a6d0f3d-eab1-49af-9588-6f5fdcf1e7f8\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:54.78621745Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"4cc89a21-5cff-420e-8ab2-13266724919f","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27867,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:54.786389573Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"454.144362ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4cc89a21-5cff-420e-8ab2-13266724919f\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4cc89a21-5cff-420e-8ab2-13266724919f\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:55.269449015Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"926cd147-7e3a-47ed-bbc3-01bfad0c28d4","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_1e68cd35-156b-4759-b35d-cd686e7b1b4b /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":27537,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:08:55.269608168Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959","retry_after":"482.162635ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"926cd147-7e3a-47ed-bbc3-01bfad0c28d4\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:08:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"926cd147-7e3a-47ed-bbc3-01bfad0c28d4\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:55.78086592Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3465"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:08:59.368004168Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3483"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:11.996812884Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3506"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:11.996938127Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"72a23f32-5b34-4cf4-be90-27abd0e268ec","remote_addr":"[fd00:1122:3344:101::1]:51068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":16245491,"response_code":"204"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:12.001325745Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.086634968s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.031143225Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.031329068Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51321"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.031658414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.031672804Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.054430878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.054475279Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.073491767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.073551488Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.096287612Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.096332523Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.115562744Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.115620555Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.135321865Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.135364846Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.153710882Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.153749663Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.172303232Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.172345053Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.190130789Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.19017377Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.468588626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.468661738Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.631066093Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.631123184Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.854267778Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:12.854321389Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.06941114Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.069471881Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.249074782Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.249134413Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.417413013Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.417474603Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.511627836Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.511680457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.597439531Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.597499452Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.722017153Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.722071494Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.835557781Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.835610462Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.936416202Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:13.936461433Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.056837422Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.056905783Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.057193668Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.057380732Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179737685Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179802866Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179814397Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179822157Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179829937Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179837367Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179844757Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179850607Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179871368Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179880528Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179889538Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179898728Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179906548Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179915748Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179926999Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179938659Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179945289Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179951629Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.179956619Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.180038791Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cf343831-5101-4fd6-9a91-c97825aeb306","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized)]","latency_us":2148823,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.180315946Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"84.998887ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cf343831-5101-4fd6-9a91-c97825aeb306\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cf343831-5101-4fd6-9a91-c97825aeb306\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.266528297Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.267023286Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.267034246Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.289336512Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.289365163Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.30998953Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.31002251Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.329000727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.329038048Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.348805939Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.34884166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.367526562Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.367552362Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.386159503Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.386184913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.403855247Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.403889088Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.421974959Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.421997039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.440771353Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.440810394Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.45972706Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.45975319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.477878032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.477915633Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.48227856Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52784"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.482427513Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"41666991-fa61-479a-a229-4255c3921993","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52784","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":80,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.497036182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.497067533Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.515134724Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.515181925Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.533903877Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.533941128Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.552875205Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.552909335Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.576028696Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.576065826Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.59486354Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.594893551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.613680705Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.613717715Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.632290645Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.632320536Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.650234474Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.650288555Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.650534269Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.650697072Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740110891Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740186892Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740193442Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740199452Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740206662Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740230453Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740235373Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740239413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740243663Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740247783Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740251873Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740255493Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740267183Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740281764Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740302554Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740320864Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740336535Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740352035Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740370515Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.740492207Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0aa2e440-a6c4-4eff-85e9-a14efefd2ba0","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized)]","latency_us":473738,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:14.740674691Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"150.21727ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0aa2e440-a6c4-4eff-85e9-a14efefd2ba0\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0aa2e440-a6c4-4eff-85e9-a14efefd2ba0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.891857707Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.892246083Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.892265114Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.911631198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.911670379Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.929875472Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.929915713Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.945374267Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.945416818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.962206637Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.962243147Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.980520772Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.980558832Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.997525294Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:14.997558805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.019097717Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.019137588Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.039147643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.039199904Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.05867881Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.058715751Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.077132888Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.077176359Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.095741829Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.09578463Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.116212893Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.116254913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.134735182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.134781792Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.154836569Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.154873129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.172857869Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.172894929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.194704927Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.194744268Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.214706432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.214759733Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.232985457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.233022878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.251562787Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.251598298Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.270090216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.270160088Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.270360471Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.270495543Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361090093Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361134884Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361142854Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361148334Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361152454Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361165114Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361170725Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361177565Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361184005Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361188975Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361194095Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361199195Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361205295Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361211755Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361215895Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361220845Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361224785Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361228726Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.361233926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.361300657Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d2a1d175-2ab1-4580-875c-453c9fbaf791","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized)]","latency_us":469315,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:15.36149358Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"203.12384ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2a1d175-2ab1-4580-875c-453c9fbaf791\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2a1d175-2ab1-4580-875c-453c9fbaf791\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.565893371Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.566217607Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.566225928Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.589277567Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.589310167Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.60804024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.608067901Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.627060848Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.627095869Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.645953834Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.645984854Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.663905143Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.663931983Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.683045932Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.683080263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.700651155Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.700689326Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.71779947Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.717843861Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.736307709Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.736335089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.754564573Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.754596343Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.774523947Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.774558368Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.793101938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.793127408Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.811482834Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.811523405Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.829352322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.829397282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.847943932Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.847973842Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.866087724Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.866134765Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.88388947Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.883917041Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.902065913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.902094664Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.919846959Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.91987989Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.938100834Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.938138364Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.938393939Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:15.938559362Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028564451Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028608902Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028616782Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028621582Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028629282Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028658242Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028664912Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028672843Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028680923Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028685703Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028690463Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028695183Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028699553Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028704243Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028710523Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028716573Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028722624Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028727033Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028733424Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.028775085Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1cbb13e3-c2f7-4434-a65d-a3fce082718f","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized)]","latency_us":462784,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.028979628Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"179.682272ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1cbb13e3-c2f7-4434-a65d-a3fce082718f\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1cbb13e3-c2f7-4434-a65d-a3fce082718f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.209330412Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.209617317Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.209626627Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.231070118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.231110349Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.251091624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.251129465Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.270887236Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.270925116Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.288756173Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.288795144Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.307820982Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.307852443Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.325773531Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.325817882Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.344669677Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.344704827Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.363647424Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.363685584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.38257886Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.382623541Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.401010087Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.401048548Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.418838704Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.418874395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.436440027Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.436472497Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.45403547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.454100581Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.471931438Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.471968288Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.489470699Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.48951013Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.50695099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.50699855Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.526397305Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.526441406Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.547392368Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.547431129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.567671319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.567711149Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.587611563Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.587659163Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.587880458Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.5880137Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679613817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679670128Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679677138Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679683169Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679690949Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679696529Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679714589Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679720959Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679727049Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.679735019Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67974121Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67974821Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67975434Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67975978Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67976535Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67977122Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67977973Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67978847Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.67979532Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.679931553Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0ef81904-7474-4c1d-b1e1-6058599b167c","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized)]","latency_us":470515,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:16.680128716Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"200.648582ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ef81904-7474-4c1d-b1e1-6058599b167c\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0ef81904-7474-4c1d-b1e1-6058599b167c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.881362331Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.881695967Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.881713078Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.902830353Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.902880283Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.920845893Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.920891634Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.941592671Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.941640112Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.960353684Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.960419366Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.978639449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.97868586Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.997634967Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:16.997678848Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.017396018Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.017441389Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.035793445Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.035840816Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.055671238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.055717679Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.074089955Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.074141266Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.093120583Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.093167474Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.11263964Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.112690221Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.130542668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.130605289Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.150279479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.15033182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.168037624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.168088395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.187979659Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.188024339Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.20607424Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.206135631Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.225834291Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.225886282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.246121441Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.246161812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.26519882Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.265248791Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.265521196Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.265687989Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359495746Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359545136Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359552287Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359558077Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359563677Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359568677Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359592197Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359598297Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359603678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359609298Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359614888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359619938Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359624918Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359630068Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359635398Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359641878Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359647298Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359652698Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359658109Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.359782951Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"186a4d81-c06d-4334-8e43-e54c20275122","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized)]","latency_us":478352,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:17.359971354Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"169.481142ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"186a4d81-c06d-4334-8e43-e54c20275122\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"186a4d81-c06d-4334-8e43-e54c20275122\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.530376891Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.530745148Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.530754198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.551526757Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.551574868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.571181147Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.571220097Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.590118923Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.590161154Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.609963236Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.610014946Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.628946813Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.628983954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.64738169Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.647422971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.666126753Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.666164044Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.684415518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.684462658Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.703412045Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.703448096Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.721798742Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.721839572Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.739861812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.739896063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.758034655Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.758072096Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.776468353Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.776517234Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.794543454Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.794581125Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.812494463Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.812529584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.830554594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.830598605Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.849025572Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.849069733Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.86751047Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.867573252Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.885705904Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.885747584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.903938707Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.904006479Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.904192402Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:17.904323554Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000602685Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000668226Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000676576Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000683956Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000690046Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000697417Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000715337Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000721397Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000726997Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000732657Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000739637Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000745427Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000751167Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000756958Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000764008Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000769428Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000776658Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000783548Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.000789498Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.00090232Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1efe2abd-2de7-4429-940d-759916a5ef2f","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized)]","latency_us":470418,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.001111804Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"267.152065ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1efe2abd-2de7-4429-940d-759916a5ef2f\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1efe2abd-2de7-4429-940d-759916a5ef2f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.269294558Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.270066532Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.270076442Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.290372633Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.290426834Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.30935597Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.309404841Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.328284386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.328330667Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.346447439Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.346519761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.365407306Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.365452947Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.383821953Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.383877214Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.402738449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.40279706Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.421152996Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.421209477Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.441945916Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.441990467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.460989204Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.461034555Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.480050313Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.480094474Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.498803266Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.498849307Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.518901513Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.518964304Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.539404327Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.539456848Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.559523355Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.559571015Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.578540812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.578593284Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.597570311Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.597615231Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.618767647Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.618828708Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.637934038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.637978188Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.658769138Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.658834989Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.659060553Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.659220826Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756570715Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756629146Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756636476Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756643946Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756648446Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756654207Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756678147Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756684467Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756691327Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756697027Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756702757Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756709717Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756713857Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756721228Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756726768Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756733708Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756739628Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756745028Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.756751888Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:18.7568551Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"cc07fb8c-c72a-4827-b33a-ad37465eeaae","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized)]","latency_us":487467,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:18.757058224Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"379.440254ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cc07fb8c-c72a-4827-b33a-ad37465eeaae\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cc07fb8c-c72a-4827-b33a-ad37465eeaae\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.137524123Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.137869289Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.137877039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.158906803Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.158956644Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.177693997Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.177741648Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.195641875Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.195692066Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.213667486Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.213719347Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.234118079Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.23416544Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.253534104Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.253582965Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.273167353Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.273217274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.291645531Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.291691552Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.311095377Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.311141807Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.329430712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.329473373Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.34840976Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.34845089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.366845797Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.366886998Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.385262734Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.385306995Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.404751001Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.404798241Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.424316838Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.424363039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.443640441Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.443685692Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.462551877Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.462592448Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.482911229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.48297198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.503846551Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.503894032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.524159752Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.524216853Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.524442357Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.524587259Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617069922Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617135264Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617144044Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617149664Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617155324Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617162574Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617180295Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617186755Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617194135Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617200095Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617207655Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617213715Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617220595Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617224885Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617230316Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617237465Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617243106Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617248556Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617254126Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:19.617386528Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"2aef4c5b-2d82-4384-bdf5-26322225091b","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized)]","latency_us":479758,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:19.617600102Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"574.977829ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2aef4c5b-2d82-4384-bdf5-26322225091b\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2aef4c5b-2d82-4384-bdf5-26322225091b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.193101016Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.193433522Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.193441692Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.215330841Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.215374282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.234271087Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.234315308Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.254058179Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.254101179Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.273308361Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.273360642Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.291952442Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.292000393Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.311094712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.311133343Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.329714913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.329754034Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.348467876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.348505487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.36728349Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.367326161Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.385944612Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.385986023Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.405308246Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.405350057Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.424264213Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.424308154Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.44266394Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.44270609Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.464492457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.464531758Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.484233518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.484272059Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.503337638Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.503375048Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.522644741Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.522683421Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.542102716Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.542146157Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.562296485Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.562335366Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.580908146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.580955617Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.5811566Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.581282492Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678692463Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678751414Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678759424Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678765294Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678770884Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678778134Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678783864Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678807435Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678814535Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678820485Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678827985Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678833715Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678838065Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678845176Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678850856Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678854926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678862306Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678868156Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.678873616Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:20.678974388Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"61d5559e-2423-4fe8-a79c-130e8adba4d1","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized)]","latency_us":485769,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:20.679158621Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"685.969766ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"61d5559e-2423-4fe8-a79c-130e8adba4d1\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"61d5559e-2423-4fe8-a79c-130e8adba4d1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.366469881Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.366796517Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.366803537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.389092793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.389141274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.409591818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.409633948Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.432190429Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.43224884Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.451237487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.451293898Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.471740302Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.471783212Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.492305157Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.492359298Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.512224691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.512264322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.53299817Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.533040241Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.554034304Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.554071564Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.574195792Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.574236172Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.594542703Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.594582584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.613180924Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.613215575Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.633716499Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.63376592Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.656713598Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.656757409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.677235242Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.677272623Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.697500672Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.697540233Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.717311624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.717353775Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.735691351Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.735750052Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.754160159Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.754205889Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.773433641Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.773488852Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.773741387Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.773910359Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873734553Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873798424Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873806974Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873814594Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873820165Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873826085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873832915Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873849625Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873855265Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873858835Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873862645Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873867945Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873871545Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873875286Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873880795Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873884576Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873891396Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873895206Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.873900406Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:21.873944187Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"bca1d3de-a51d-465f-8ce9-80e0970581b5","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized)]","latency_us":507384,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:21.87413051Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"482.86218ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bca1d3de-a51d-465f-8ce9-80e0970581b5\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bca1d3de-a51d-465f-8ce9-80e0970581b5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.358630118Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.359551814Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.359559994Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.384750682Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.384804053Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.407101669Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.40715414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.427017202Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.427069454Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.446725193Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.446781654Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.465757951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.465803042Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.485090234Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.485135375Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.504700133Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.504742594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.52367678Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.523726391Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.542939602Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.542989943Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.56362374Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.56366366Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.58281663Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.582860621Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.602041052Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.602083723Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.620826806Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.620875027Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.641401211Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.641444232Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.661858045Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.661902166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.681556895Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.681596295Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.700008772Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.700046983Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.718387859Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.71843547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.736552741Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.736593042Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.757933201Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.758002762Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.758230267Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.758362169Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852269187Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852334478Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852339609Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852345119Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852350689Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852355029Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852360729Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852386449Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.85239199Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.85239903Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.85240428Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.85241039Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.85241678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.85242559Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852432Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.8524385Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.85244313Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852447591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.8524519Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:22.852541732Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9ec68752-d1d5-454b-b739-40566188b006","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized)]","latency_us":493815,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:22.852745556Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"650.68261ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9ec68752-d1d5-454b-b739-40566188b006\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9ec68752-d1d5-454b-b739-40566188b006\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.504456344Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.50534651Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.50535954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.529412667Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.529458178Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.550591374Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.550636504Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.570319814Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.570368655Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.589754919Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.58980662Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.608782237Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.608824428Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.627639783Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.627688613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.646584039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.64662524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.666232468Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.666274919Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.687842251Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.687899473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.708551459Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.70859296Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.729621204Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.729660954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.750276201Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.750319811Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.769221267Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.769266818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.78963894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.789687191Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.810033812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.810084883Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.830065718Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.830118609Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.849579555Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.849626696Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.868639173Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.868700805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.887642391Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.887689642Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.907728418Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.907774709Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.907980662Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:23.908105545Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003653692Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003730573Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003736904Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003743424Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003748074Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003754374Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003759284Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003779114Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003783754Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003788594Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003795085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003801315Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003806375Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003811105Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003817175Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003823655Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003828205Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003832925Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.003837395Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.003842386Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b36a1902-2e61-40ca-ac07-02c669aae364","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized)]","latency_us":499281,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:24.004022629Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"599.161842ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b36a1902-2e61-40ca-ac07-02c669aae364\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b36a1902-2e61-40ca-ac07-02c669aae364\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.604457086Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.604847453Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.604857203Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.627670468Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.627718629Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.647847027Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.647892268Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.667852432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.667909603Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.68746211Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.687519581Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.707077929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.70712147Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.725877213Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.725922674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.742653961Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.742697092Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.760863614Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.760905415Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.782286365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.782335346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.802242459Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.80228874Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.822907377Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.822950638Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.841837853Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.841880354Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.860312391Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.860359432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.879123956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.879162946Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.898442229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.898485509Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.918942093Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.918983144Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.938091603Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.938131894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.956645643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.956688754Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.977469123Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.977515584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.99704137Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.997096731Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.997282435Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:24.997418637Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095218875Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095285406Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095292506Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095298276Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095304306Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095311856Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095319596Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095346247Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095353437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095361487Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095368327Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095377427Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095384738Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095390318Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095396188Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095401818Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095407548Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095416208Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095430548Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:25.09555431Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"814e3ddc-a4b7-48a1-8b5d-cb88abf0cca6","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized)]","latency_us":490961,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:25.095744694Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.264639866s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"814e3ddc-a4b7-48a1-8b5d-cb88abf0cca6\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"814e3ddc-a4b7-48a1-8b5d-cb88abf0cca6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.362353996Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.362707072Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.362716413Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.385868994Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.385916374Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.404483035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.404526255Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.425351915Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.425405796Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.444697529Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.44474Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.463501333Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.463545064Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.481222478Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.481265299Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.500923138Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.500961278Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.521264299Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.521306Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.546689221Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.546755992Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.568957566Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.568996137Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.589020423Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.589058634Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.610020076Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.610058467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.627076129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.62712442Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.645377754Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.645418785Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.665532242Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.665568723Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.686196639Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.686240069Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.706335357Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.706377067Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.727793638Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.727835339Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.746965348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.747005069Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.767130686Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.767184118Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.767410122Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.767554984Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863453038Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863501209Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863507819Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863512439Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863515829Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863520009Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863523249Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863527579Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.86354211Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863547719Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863552359Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.86355655Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.86356194Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.8635663Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.86357145Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.86357576Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.86357984Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.8635848Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.86358898Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:26.863690732Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"6ec97c01-c043-415b-821a-9c8cc67ab73f","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized)]","latency_us":501219,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:26.863877025Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"932.009198ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6ec97c01-c043-415b-821a-9c8cc67ab73f\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6ec97c01-c043-415b-821a-9c8cc67ab73f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.796522644Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.796901561Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.796908741Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.819297019Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.81933557Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.838123193Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.838157644Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.856230375Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.856273216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.874923127Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.874963348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.894341382Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.894372473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.913006994Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.913041894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.931786977Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.931833798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.952723929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.95275668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.97189427Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.971933621Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.991180883Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:27.991217403Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.013121322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.013169823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.034735716Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.034774887Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.056400951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.056444482Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.074964121Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.075000272Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.094761503Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.094797903Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.114050586Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.114086736Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.122601067Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.304142322s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.134157793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.134198563Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.153761741Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.153793992Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.174296136Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.174326646Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.19309882Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.193133931Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.193325454Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.193447756Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288763829Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.28882073Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288827651Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288835071Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288840911Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288848711Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288873021Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288880612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288886341Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288892022Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288898962Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288905812Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288911522Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288917312Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288922882Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288928322Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288933852Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288940573Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.288946013Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:28.289035044Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4132997f-98c9-4ee9-84c1-79273c7467c9","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized)]","latency_us":492378,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:28.289264378Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"939.279519ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4132997f-98c9-4ee9-84c1-79273c7467c9\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4132997f-98c9-4ee9-84c1-79273c7467c9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.229569344Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.22992152Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.22993184Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.252299267Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.252347808Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.271939506Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.271977787Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.293413488Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.293455008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.312426685Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.312473386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.334424436Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.334465577Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.355946669Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.35599557Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.375828082Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.375882203Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.396586721Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.396637552Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.417293519Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.417334939Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.436850386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.436896847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.456909792Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.456950043Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.478931823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.478974894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.499951997Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.500002648Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.520890649Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.52092969Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.542487893Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.542524573Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.562266814Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.562304825Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.582768958Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.582809579Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.602504689Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.60256211Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.621276543Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.621314303Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.640525445Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.640577465Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.64083602Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.641001063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739312489Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.73936071Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.73936749Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.73937185Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.7393753Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.73937863Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.73938971Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.7393942Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.73939943Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739403501Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.73940769Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739412741Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739416911Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739420971Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739426001Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739430471Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739434591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739438621Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739443451Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:29.739535643Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"507e4c08-17b0-4607-8bfc-672a3358f93c","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized)]","latency_us":509867,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:29.739745736Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.39848719s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"507e4c08-17b0-4607-8bfc-672a3358f93c\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"507e4c08-17b0-4607-8bfc-672a3358f93c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.13985711Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.140202956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.140211537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.16350117Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.163547621Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.185037803Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.185087254Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.206951622Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.206994563Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.22650415Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.226562331Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.24565195Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.245694061Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.264945853Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.264989393Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.283818098Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.283862089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.302683003Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.302730044Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.321458637Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.321496368Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.341080505Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.341117906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.360244926Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.360285726Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.379302234Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.379346485Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.398003457Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.398056617Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.417072815Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.417113986Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.436184705Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.436225216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.456395324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.456436965Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.478282433Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.478324244Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.498103035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.498153156Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.517824345Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.517870216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.540310115Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.540375086Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.5406089Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.540776393Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639145831Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639192352Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639197842Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639201252Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639205432Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639209712Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639228562Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639233182Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639237472Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639241582Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639245662Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639250653Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639254743Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639258883Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639263763Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639267823Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639271763Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639275873Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639279793Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:31.639398115Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"0e813884-0e18-4a62-a41f-39d9ccac723b","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\", TimeNotSynchronized), (\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\", TimeNotSynchronized), (\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\", TimeNotSynchronized), (\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\", TimeNotSynchronized), (\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\", TimeNotSynchronized), (\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\", TimeNotSynchronized), (\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\", TimeNotSynchronized), (\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\", TimeNotSynchronized), (\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\", TimeNotSynchronized), (\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\", TimeNotSynchronized), (\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\", TimeNotSynchronized), (\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\", TimeNotSynchronized), (\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\", TimeNotSynchronized), (\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\", TimeNotSynchronized), (\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\", TimeNotSynchronized), (\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\", TimeNotSynchronized), (\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\", TimeNotSynchronized), (\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\", TimeNotSynchronized)]","latency_us":499457,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:31.639600839Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.004534692s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0e813884-0e18-4a62-a41f-39d9ccac723b\", \"content-length\": \"124\", \"date\": \"Thu, 30 May 2024 04:09:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0e813884-0e18-4a62-a41f-39d9ccac723b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5269a058-580b-4ad2-a686-a68f8b8e0982, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd14d7c8-e7c7-4706-93f1-737bc0db512a, kind: Service { id: 37aef7a7-53c8-4b7f-aab3-5292349f3f5d }, name: Name(\"ntp-37aef7a7-53c8-4b7f-aab3-5292349f3f5d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 65])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: 72c3976d-68e9-463a-be70-3de38abfe035, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: a84ccf3e-ca81-499f-b526-5c900c243456, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9112ad3c-6c15-4601-bf24-48148ee77690, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 5cc3fd7a-881a-45bf-9524-016003ecf668, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8f3604d6-5231-4728-9097-caf093b736ed, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: ca8c551e-b7e2-4add-a459-c77679e168ba, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: c2209168-b7ee-4e88-824f-9effc24609ea, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2bbcf07c-7614-4e39-9570-91a5c47d5232, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External } } } }, OmicronZoneConfig { id: df8c46fb-39e3-4705-bde9-709842a66be5, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External } } } }, OmicronZoneConfig { id: dbc6937e-eabc-4bd1-922e-b561820024c2, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15faa041-d273-4f44-b56e-98664dd6c144, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External } } } }, OmicronZoneConfig { id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f1da100-e042-4583-8561-6eff4c72a835, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3388152a-babe-4c26-b4bb-14448f905920, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.007708922Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.008208819Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.008215345Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5269a058-580b-4ad2-a686-a68f8b8e0982, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.031482414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.031523224Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 448a7523-abaa-4f37-8ea0-88fd1a15dcfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.05098631Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.051020771Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a5f19f92-dd9d-4d5d-b88b-df21ddc95063, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.070007627Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.070067759Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1e68cd35-156b-4759-b35d-cd686e7b1b4b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.089751843Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.089791809Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fef621b3-c922-4648-a07b-e45ce7dbe3e0, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.108339348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.108394269Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 72c3976d-68e9-463a-be70-3de38abfe035, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.128140883Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.128223375Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a84ccf3e-ca81-499f-b526-5c900c243456, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.146829725Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.146874145Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9112ad3c-6c15-4601-bf24-48148ee77690, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.166264116Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.166310217Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56096d63-69cd-4c67-9c8f-fe757a4566d3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.185357582Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.18541267Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.203795137Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.203837532Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ca8c551e-b7e2-4add-a459-c77679e168ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.222120768Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.222178547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b5ac113d-4f55-4c3e-8718-a1f7414bba5f, dataset_name: DatasetName { pool_name: ZpoolName { id: 3022fa0e-6566-4854-a1a9-1e0c5908d5b0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.240167215Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.240224225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f5dbf3d-e4c0-438a-82cb-4d43a6453128, dataset_name: DatasetName { pool_name: ZpoolName { id: 1793c88e-7478-4741-8254-860fae35146b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.257800197Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.257855783Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: df8c46fb-39e3-4705-bde9-709842a66be5, dataset_name: DatasetName { pool_name: ZpoolName { id: f61d361f-e4c0-48df-9191-ff705041ee76 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.275432035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.275482755Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbc6937e-eabc-4bd1-922e-b561820024c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.293304962Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.29337325Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 15faa041-d273-4f44-b56e-98664dd6c144, dataset_name: DatasetName { pool_name: ZpoolName { id: 5d12239f-a80e-4f75-83d4-ebef24b6fea1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.311627839Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.311679196Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 47b0a86a-e796-41a6-af75-fb0dfd4a86c1, dataset_name: DatasetName { pool_name: ZpoolName { id: a168fba6-15db-4743-814f-b76d3daf999f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.329773334Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.329828038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dbfa33e5-1797-4930-9f90-903e48c8cf4c, dataset_name: DatasetName { pool_name: ZpoolName { id: 2efcf3a6-ef7f-48c2-8713-0dd19fe385d1 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.347788288Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.349778435Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f1da100-e042-4583-8561-6eff4c72a835, dataset_name: DatasetName { pool_name: ZpoolName { id: dc70dd36-578e-4d7e-9ca2-11c36573efc3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.366131974Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.366183655Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3388152a-babe-4c26-b4bb-14448f905920, dataset_name: DatasetName { pool_name: ZpoolName { id: 8f547a7a-6177-4ca0-82ce-6646cf05d8a6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.384114083Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.384180634Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.384405688Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.384533622Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.481336656Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.481691141Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.483374078Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.541644318Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.543892782Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.543985494Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.544006915Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.544044051Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.544069194Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.544118424Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.748784745Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.749043171Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.749100378Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.749139725Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.749188392Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.750497727Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.763853554Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.764000727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.765325024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.765524346Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.77133548Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.776250221Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.781109183Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.78615894Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.791066617Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.795887069Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.801204494Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.920365341Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.925295204Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.930031398Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.934185021Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.938956081Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.944118235Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.949157824Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.95408343Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.958341625Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.962382463Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.968031628Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.973711916Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.979537459Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.986559507Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.992073892Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:33.997908768Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.002930017Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.008569908Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.015560447Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.022105636Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.074827297Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.134865972Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.178510422Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.23375575Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.36700947Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.368804144Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.368825566Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.462820521Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 213, 208]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.462914375Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.466305369Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.466340611Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.595244779Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 248, 105]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.595307489Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.600166238Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.600180469Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.70707831Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 229, 173]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.707162977Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.711542598Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.711560204Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.793147661Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 170, 13]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.793216597Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.795739949Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.795761699Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.889564376Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 244, 140]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.889649907Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.890325229Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.89035689Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.890715505Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.890730271Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.890929615Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.890945206Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.891442749Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.894230249Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.894250129Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.897533183Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.900555981Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:34.900567484Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.008293752Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.008375091Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.008620129Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.008640499Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"Configuring new Omicron zone: oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.008975125Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.016931958Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.02581985Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.029328149Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.032630337Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.032649622Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.032918352Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.103057297Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.108623557Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.114278053Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.121915769Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.127328331Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.182654195Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:35.225645001Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.024309807Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.025122568Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.116334931Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.116674847Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.130162126Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.130307419Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.198759883Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.198934786Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.303660286Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.303844846Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 58.282989ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.385437652Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 27.182519ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.400554918Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 48.727239ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.414593599Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.415914778Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.416221056Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.424599039Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.424820237Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 40.703986ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.434908185Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.435124649Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.435183181Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.440053777Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 50.847056ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.454274678Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 110.327381ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.463261021Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.463355298Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 130.745234ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.495754963Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 100.001701ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.511383583Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 224.679238ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.526101587Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 25.213985ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.571361887Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 182.376295ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.590716465Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.591224403Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.591527399Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 75.569759ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.616913196Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.648572298Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 211.67388ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.658625679Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.695473873Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 226.54715ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.718268287Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 155.863816ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.733142738Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.735849527Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 43.325542ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.750302184Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.783513166Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:36.792428957Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 362.050078ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.86051826Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 312.79208ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.930977501Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 444.615301ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:36.98713408Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 140.715183ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.006881618Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 350.576769ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.026613624Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 53.574183ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.04168652Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 46.419762ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.056634354Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 61.932962ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.07014736Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 72.729913ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.084261309Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:37.084430896Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 484.089242ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.109022391Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 95.699721ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.143758602Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 123.098452ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.158893444Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 41.328842ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.194267229Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 107.815251ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.362341567Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 139.58482ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.393197735Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 54.637509ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.41992269Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 496.508519ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.44309009Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 688.249307ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.473880672Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 740.43603ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.498049151Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 108.897479ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.526648377Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 1.025746671s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.627462494Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 172.560933ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.710870244Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 255.216767ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.762657687Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 50.701607ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.798520075Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 276.389493ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.815912934Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 293.078898ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.833869312Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 103.942307ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.850451026Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 74.0002ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.862983583Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 860.342893ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.877195686Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 356.901189ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.89079677Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 576.959442ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.904874737Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 148.752316ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.920996035Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 147.032266ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.950772761Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 942.012413ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.965512824Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 146.0423ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:37.981311135Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 242.894579ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.035221371Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 353.70743ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.084583672Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 436.421104ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.107616288Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 280.722373ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.124123538Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 510.975187ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.140401891Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 321.786737ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.154094372Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 1.174166098s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.180349648Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 1.47869135s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.25378912Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 892.510534ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.267631508Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 924.993436ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.294498709Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 286.255647ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.420622023Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 1.130938679s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.450963061Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 941.056349ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.491475902Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 660.637276ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.507730686Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 967.859365ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.561108099Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 696.032544ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.667135672Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 734.535082ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.678824351Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 422.829444ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.754719122Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 646.48524ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:38.833478065Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 1.049583483s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.014481344Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 1.230344511s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.256997648Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 895.788092ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.348106401Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 914.384513ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.414513302Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 1.479500974s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.432111471Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 1.316533214s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.448809838Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 852.680851ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.462479084Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 663.921197ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.474301206Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 500.389112ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.491112108Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 938.527215ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.506778517Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 1.151459267s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.545450529Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 1.269967048s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.599596564Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 538.520673ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:39.815793187Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 653.260749ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.007401554Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 1.390240281s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.079389846Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 1.12831668s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.152056866Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 1.475465615s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.259867196Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 577.398411ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.330680689Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 1.310004438s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.344171059Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 831.907573ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.368024484Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 1.123647062s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.463562332Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 1.401910087s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.502988125Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 1.233907356s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.71158688Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 1.499597834s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.782065406Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 834.604763ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.797766101Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 999.732493ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.885114521Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 830.228664ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.923438639Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 658.211269ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:40.938517759Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:40.979510999Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:40.979823005Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 570.909272ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.215905009Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 805.271274ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.296797507Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 64.437485ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.323147032Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 90.983006ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.406282219Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 1.072446456s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.484797552Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 129.972654ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.509298727Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 1.412841696s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.600829782Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 648.008811ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.613837313Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 1.111160708s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.647166982Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 321.773409ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.660211167Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 1.191343373s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.680408845Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 519.473458ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.761434229Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 818.664964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.789466102Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 1.370765541s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.810082451Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 708.101964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.904578765Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 981.935525ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.924725482Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 1.096592079s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:41.961904124Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 623.043731ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.000142966Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 1.32102673s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.119780384Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 721.635749ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.312327965Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 1.05685519s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.336169978Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 817.46484ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.355703183Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 861.935626ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.578696501Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 1.055279391s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.632092121Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 704.092285ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.66121222Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 1.108185608s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.682464718Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 1.453681446s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.799817143Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 898.830301ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.920465035Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 1.110851834s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:42.956029708Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 1.357961059s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.044537517Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 1.337739575s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.062259553Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 913.49333ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.079272315Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 1.130482903s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.191634859Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 898.537979ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.255706615Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:43.257869389Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:43.258646665Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:43.280492022Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:43.280657626Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 528.193317ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.431530903Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 669.50275ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.459332411Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 747.605234ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.478826477Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 849.799652ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.498335021Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 30.462365ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.630908408Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 55.95698ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.648027211Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 65.073338ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.675723182Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 822.545383ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.704032434Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 84.377906ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.719675765Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 232.76501ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.757930462Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 820.364324ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.821798135Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 652.914608ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.836579776Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 199.682156ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:43.849509403Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 779.468335ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.010548166Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 262.953963ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.048341675Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 634.743071ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.083760841Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 596.547892ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.108882316Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:44.130128108Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46565"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:44.133051542Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"860e589c-3bc9-4668-9f49-0993c26aac09","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46565","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2595,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 764.692361ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.138812612Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 991.9501ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.157802746Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 728.201147ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.183513285Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 953.272166ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.27845999Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 1.320351854s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.322081599Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 678.670181ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.34028601Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 570.354685ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.354861058Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 1.045554162s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.369158561Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 770.163574ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.41644891Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 1.135771584s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.429561184Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 1.298907021s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.504478506Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:44.524003199Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:44.524078951Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 768.579641ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.544686069Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 1.302200348s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.655513189Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 1.045968643s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.719925627Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 677.87633ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.733455032Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 46.076772ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.779854446Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 1.142587522s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.801527469Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 122.492208ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.836398041Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 908.118633ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.921285871Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 1.395203354s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.932945588Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 1.146018901s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.943216366Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.951378324Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"5.75631181s"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 258.370068ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:44.97250114Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 1.236574262s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.03150843Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 1.007145229s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.163574721Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 569.465418ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.198894909Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 404.94563ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.241823441Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 798.476805ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.252825089Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 1.050607342s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.326954649Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 1.464288561s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.426007881Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 1.152145571s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.434956809Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 505.401744ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.580337917Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 512.22153ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.675469959Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 938.601899ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.691741386Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 1.437140528s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.781096569Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 639.3744ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.794237802Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 804.592651ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.818897377Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 1.250933305s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.848190119Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 633.350646ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.958046299Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 1.4018423s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:45.969490176Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 1.14091228s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.062717087Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 1.455196019s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.097468888Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 846.609678ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.107641197Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 775.55804ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.184349336Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 1.146638778s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.198534962Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 1.36879945s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.281476931Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 641.701427ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.343077195Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 671.370828ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.391358527Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 899.0721ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.447432884Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 593.218131ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.601475415Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 1.382798795s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.612988993Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 748.421541ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.636350804Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 963.179951ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.647511744Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:46.71994716Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:46.7202117Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:46.813752025Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:46.813867105Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 886.075504ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.91673049Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 521.343552ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:46.979489438Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 1.460398301s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.00152379Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 1.318746725s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.023107779Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 73.598896ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.079130842Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 1.118769678s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.104421Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 900.692785ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.131078822Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 80.307814ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.20007636Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 1.41386001s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.29302132Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 1.290369975s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.309664677Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 28.655784ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.321453068Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 534.074029ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.33637054Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 166.69843ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.347752452Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 698.415681ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.36221479Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 1.16504985s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.381830747Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 62.144996ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.393146457Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 915.184323ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.414932568Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 652.861053ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.437243152Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 149.135319ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.480443915Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 1.286258537s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.524336736Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 209.97367ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.540447442Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 1.030907128s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.573737266Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 983.671549ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.632422344Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 564.889054ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.652027973Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 1.130265467s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.672521145Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 503.708396ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.767637744Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 1.289466428s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.820876952Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 530.841919ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:47.887950121Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 1.181212326s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.013878835Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 997.245865ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.048443841Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 1.22752373s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.081665848Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 594.773382ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.106445883Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 637.830519ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.230984865Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 734.957185ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.242563114Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 787.388137ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.286395254Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 1.158096461s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.345248007Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 865.82541ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.359810406Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 689.58761ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.435746671Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 918.873858ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.478577137Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 1.199332491s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.562218594Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 524.612464ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.616923269Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 1.025599712s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.62954996Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 1.13854945s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.639891977Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 1.354052651s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.715178649Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 840.815493ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.725893425Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 501.394133ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.819923991Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 1.361557867s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.838519243Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 1.013879846s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.893600123Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 1.155591293s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:48.996167871Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 938.409753ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.059086937Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 635.646543ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.089134393Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 766.703793ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.126654377Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 594.177477ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.139557356Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 1.189502564s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.156895188Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 943.614982ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.214057559Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 615.212462ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.244219965Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 1.42026195s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.329296353Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 581.127713ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.34347397Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 989.743864ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.412823624Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 534.101052ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.517886523Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 1.104072407s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.578843564Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 619.481153ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.667763182Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 918.856498ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.744117907Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 916.980478ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.753651981Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed\") failed: Property not found. retry in 1.422182157s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.772474272Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 874.500751ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.79104328Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 647.099935ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.869359635Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c\") failed: Property not found. retry in 1.285795598s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.903047661Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 1.282194645s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.92045043Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 869.440346ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:49.935829476Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 844.018272ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.01294227Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 1.469282744s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.063211307Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 599.371814ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.081632208Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 833.917512ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.163933299Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1\") failed: Property not found. retry in 1.330294949s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.173493819Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 822.663084ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.210983751Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea\") failed: Property not found. retry in 1.37834972s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.300004387Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab\") failed: Property not found. retry in 1.193585855s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.360215224Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2\") failed: Property not found. retry in 888.845831ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.415007907Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f\") failed: Property not found. retry in 1.451924547s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.527975746Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 1.349797717s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.674382493Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 1.109400104s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.682003795Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 1.317120031s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.688656307Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920\") failed: Property not found. retry in 1.027023418s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.695259201Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 1.430435804s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.702716673Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232\") failed: Property not found. retry in 922.983729ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.764944454Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 1.131068541s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.819818059Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128\") failed: Property not found. retry in 1.321184035s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:50.868566247Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5\") failed: Property not found. retry in 702.098585ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:51.01119887Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835\") failed: Property not found. retry in 778.696502ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:51.044158989Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 646.823947ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:51.231657596Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144\") failed: Property not found. retry in 718.497059ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:51.545738815Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 900.055717ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:51.815233027Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 972.006972ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:51.897071664Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 1.33461601s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:51.970440305Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 916.175969ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:52.027042158Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 1.293641354s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:52.050241999Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 745.728542ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:52.156206081Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3\") failed: Property not found. retry in 1.214216716s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:52.736493331Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 536.596522ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:52.893748632Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba\") failed: Property not found. retry in 1.391290787s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:52.926788707Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f\") failed: Property not found. retry in 1.085264114s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:52.960788829Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 543.648649ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:53.325111201Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 977.492767ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:53.365753348Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 620.681373ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:53.450693877Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4\") failed: Property not found. retry in 1.425143519s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:53.885574336Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 1.414024213s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:54.085554548Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 580.56585ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:54.360614813Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 1.345960923s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:54.955605745Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 641.860237ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:55.514386794Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668\") failed: Property not found. retry in 966.111019ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:56.168699568Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48\") failed: Property not found. retry in 627.375251ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:56.316099876Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168703705Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_pantry_3ef0ace2-9c93-43ed-bb5d-824e6b6b0cab"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168765009Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_external_dns_6d9390f7-43dc-4e13-9714-d2a358e2f5d4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168769987Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_nexus_5cc3fd7a-881a-45bf-9524-016003ecf668"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168775209Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_pantry_c2209168-b7ee-4e88-824f-9effc24609ea"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168796612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_pantry_2bbcf07c-7614-4e39-9570-91a5c47d5232"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168800288Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_b5ac113d-4f55-4c3e-8718-a1f7414bba5f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168806907Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_1f5dbf3d-e4c0-438a-82cb-4d43a6453128"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168812982Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_df8c46fb-39e3-4705-bde9-709842a66be5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.16881647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_dbc6937e-eabc-4bd1-922e-b561820024c2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168819807Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_15faa041-d273-4f44-b56e-98664dd6c144"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168822948Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_47b0a86a-e796-41a6-af75-fb0dfd4a86c1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168827963Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_1f1da100-e042-4583-8561-6eff4c72a835"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168831488Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_3388152a-babe-4c26-b4bb-14448f905920"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168834863Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_crucible_dbfa33e5-1797-4930-9f90-903e48c8cf4c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168838145Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_external_dns_56096d63-69cd-4c67-9c8f-fe757a4566d3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168841313Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_nexus_c27f5e00-47bb-499a-ad9e-1146fabe9d48"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168844651Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_clickhouse_ca8c551e-b7e2-4add-a459-c77679e168ba"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.16896035Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_nexus_d5da8fa4-2f8d-4c39-9334-8e19330e172f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.168967072Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054","zone":"oxz_oximeter_8f3604d6-5231-4728-9097-caf093b736ed"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.169698013Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.170450732Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.172316882Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"ff98a91e-ddf8-48a3-900a-f0001bd93aff","remote_addr":"[fd00:1122:3344:101::1]:51321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":24129020,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.190648655Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1237"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.190662371Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.190810283Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.194958912Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.198407051Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:57.23052244Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:818"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:58.134142246Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:58.246882382Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:58.371717651Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:58.460349803Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:58.613182334Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:09:58.67777256Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"4.132072036s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:58.856721032Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:59.089941462Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:59.356260965Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:09:59.576399575Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:00.195027268Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:00.735467935Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:01.574798009Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:02.503777761Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:10:02.856355572Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"16.367731676s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:03.037314077Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:03.651994069Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:05.420951697Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/4559c8be-8414-459c-b1cf-c6595c790cca/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:07.087735404Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"786b9bf7-8848-440f-8c20-02603c8df42e\", \"content-length\": \"173\", \"date\": \"Thu, 30 May 2024 04:10:09 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"dd908086-f372-434c-b66b-6e5770f567d8\\\"\", request_id: \"786b9bf7-8848-440f-8c20-02603c8df42e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:09.394400688Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"28be8154-51d1-4d6b-a248-6a279f3fa0f2\", \"content-length\": \"173\", \"date\": \"Thu, 30 May 2024 04:10:12 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"dd908086-f372-434c-b66b-6e5770f567d8\\\"\", request_id: \"28be8154-51d1-4d6b-a248-6a279f3fa0f2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:12.978889546Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:14.256973869Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34412"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:14.257995007Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"e0de5274-65a1-4112-8226-b69db507a011","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34412","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":522,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f58852a7-9473-4dac-94d7-42c58b17a591\", \"content-length\": \"173\", \"date\": \"Thu, 30 May 2024 04:10:17 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"dd908086-f372-434c-b66b-6e5770f567d8\\\"\", request_id: \"f58852a7-9473-4dac-94d7-42c58b17a591\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:17.181441744Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"30a2f436-079f-4a85-800f-687944662ba5\", \"content-length\": \"173\", \"date\": \"Thu, 30 May 2024 04:10:20 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"dd908086-f372-434c-b66b-6e5770f567d8\\\"\", request_id: \"30a2f436-079f-4a85-800f-687944662ba5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:21.251687485Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"214e7633-82db-4b17-bce9-cd6e646b3625\", \"content-length\": \"173\", \"date\": \"Thu, 30 May 2024 04:10:26 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"dd908086-f372-434c-b66b-6e5770f567d8\\\"\", request_id: \"214e7633-82db-4b17-bce9-cd6e646b3625\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:26.417344844Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:880"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:29.63621834Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:29.775166479Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59783"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:29.775901467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:29.862826179Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"153c0ab1-08f5-449a-a11d-098f0e4eee40","remote_addr":"[fd00:1122:3344:101::c]:59783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":86748,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:29.86321373Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:29.863281081Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:29.865113771Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:29.865572433Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"4998552e-597e-471c-b761-661bf8c1c178","remote_addr":"[fd00:1122:3344:101::c]:59783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2501,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.000362627Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51253"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.00292495Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.003195496Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.003242766Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.003255638Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.00329043Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.003296776Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.003311758Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.003324209Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a4cd9393-7136-436d-b4f2-26e4949fb929","remote_addr":"[fd00:1122:3344:101::c]:51253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2395,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.005618614Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.251740413Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59151"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.251852333Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.333995309Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"d6ccefb3-7030-475d-ae03-685ce4d46111","remote_addr":"[fd00:1122:3344:101::c]:59151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":81850,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.334399896Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.334466778Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.334969831Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:34.335324984Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"6cd392e6-5d15-4465-a68b-1e19391a6bf3","remote_addr":"[fd00:1122:3344:101::c]:59151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1014,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.473969061Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35389"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.474273815Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.474316696Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.474359897Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.474371803Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.47438115Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.474394342Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.474411882Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.474427585Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b1a77100-1796-47bb-b188-8f47494fd7ff","remote_addr":"[fd00:1122:3344:101::a]:35389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":357,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585227528Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57593"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585448633Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585493923Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585517334Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585531209Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585551993Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585558735Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585572891Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.585578947Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"12c161a8-459c-4780-b42e-4fe86c6c76b5","remote_addr":"[fd00:1122:3344:101::c]:57593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":270,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.6537411Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.654440292Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"4501ea64-f54f-48f8-9e51-45ca32bb4a74","remote_addr":"[fd00:1122:3344:101::b]:33554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":652,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.77514584Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64685"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.85416195Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2f6e6501-7bce-4d23-bad6-0f3467938e82","remote_addr":"[fd00:1122:3344:101::b]:64685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":78670,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.854278727Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60992"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.855230199Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.855251781Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fbe78ffc-17a2-4233-958f-6a80cbe9dbc7","remote_addr":"[fd00:1122:3344:101::b]:64685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":813,"response_code":"200"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.85527459Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.855316919Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.855330447Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.855340854Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.855356576Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.855361976Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.855371642Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9bc70fa3-de6a-4d4b-a449-f90a6455b93e","remote_addr":"[fd00:1122:3344:101::b]:60992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1016,"response_code":"204"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.879749423Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/75663f09-f516-40fc-852b-77e4a30319f5/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.87977781Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/804b9788-bd41-4d87-a91b-e568478fa644/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.880855006Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.881460886Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:35.881475455Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"b0dbe8ef-45b0-42ac-8af6-4462af3c2c0b","remote_addr":"[fd00:1122:3344:101::b]:64685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2306,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.28023116Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.281179085Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"04281d89-08a6-4a9c-a142-fe6915b3b697","remote_addr":"[fd00:1122:3344:101::a]:37435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":910,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.350035006Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:65350"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.40122392Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b7e1c24b-d4c7-4e3f-b5c9-892ceeab83ee","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50967,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.401337791Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41832"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.401588323Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"6fbd8ce6-ceca-4a2f-99f6-546d03d73f87","remote_addr":"[fd00:1122:3344:101::c]:41832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":212,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.401690409Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"dc134fce-7f0e-4b17-a6c7-f1f3d6efe58e","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":223,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.514911354Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61216"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.562020988Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"37d6c157-6d31-455d-83a3-40b65b9e9c43","remote_addr":"[fd00:1122:3344:101::c]:61216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46880,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:37.562256671Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a571f148-b963-4b07-b551-cb4965e5c868","remote_addr":"[fd00:1122:3344:101::c]:61216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:44.396483526Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:44.397022617Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"712407ac-43bd-4304-b1a5-3976884d6bce","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47847","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":549,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:49.586192546Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:33087"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:10:49.59104532Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"9aaee53c-3429-427f-973b-9146d23e6754","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3929,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:05.929378258Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:05.930333197Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d94bc129-d334-4f74-ab7d-01ed30cf87d0","remote_addr":"[fd00:1122:3344:101::b]:60600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":909,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:06.211686366Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:06.259678123Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"10b4fef2-3c68-4921-8649-2079662b545f","remote_addr":"[fd00:1122:3344:101::b]:40935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47789,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:06.260008091Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0bb92e03-65ff-4dd1-9fe2-1f40c1b73a65","remote_addr":"[fd00:1122:3344:101::b]:40935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:07.471792355Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36829"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:07.471969536Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"1a4f7a7e-baa8-485c-9c39-4a546c42c161","remote_addr":"[fd00:1122:3344:101::a]:36829","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:07.670429803Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54604"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:07.671359364Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"522081dc-b3e5-4b69-956f-4588d09e6028","remote_addr":"[fd00:1122:3344:101::c]:54604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":897,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:07.76643359Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60959"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:07.810282659Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"956c783c-d66b-4c47-b4f6-792d93a6d20e","remote_addr":"[fd00:1122:3344:101::c]:60959","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43647,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:07.811104402Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9feade38-21a2-4e52-a986-cba42d3869d0","remote_addr":"[fd00:1122:3344:101::c]:60959","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":730,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:07.998187801Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:08.046676859Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"66e69e81-d4f3-4169-975f-c83657328b43","remote_addr":"[fd00:1122:3344:101::a]:47936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48280,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:08.04697614Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c97e32a3-25c7-4bba-9621-d9724d64f0fc","remote_addr":"[fd00:1122:3344:101::a]:47936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:14.51134427Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46800"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:14.511462162Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"b84a9d6a-6c89-4e74-8f1e-41754d332e20","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46800","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:19.649018124Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"b6a8d081-708a-4512-8420-ec95132b5920","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1248,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:35.726864284Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46037"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:35.727468395Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"ceb8de37-4b1a-422e-8194-632d0e78b4fd","remote_addr":"[fd00:1122:3344:101::b]:46037","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":575,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:35.812296562Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42700"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:35.865340174Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"a33810ab-d47f-4e3e-aa47-cd3438702ba7","remote_addr":"[fd00:1122:3344:101::b]:42700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52978,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:35.86567424Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1efd722d-605e-47e5-9875-dec6eb865af2","remote_addr":"[fd00:1122:3344:101::b]:42700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.367863325Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56997"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.368090678Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"4e9fd586-04c6-4d70-9ac4-3a7bdce2e008","remote_addr":"[fd00:1122:3344:101::a]:56997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":201,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.40578879Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:42983"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.405953757Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"e79e9e94-d1cd-4122-83b0-5d85a604b798","remote_addr":"[fd00:1122:3344:101::c]:42983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.544011834Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44662"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.591999726Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"bc1d7bd1-7e08-416a-ad28-290ec7a0dd98","remote_addr":"[fd00:1122:3344:101::c]:44662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47923,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.592268151Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7fc1523c-7855-46f2-b553-12cf6f83f8c3","remote_addr":"[fd00:1122:3344:101::c]:44662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.742867056Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.787892676Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2c006ca2-097e-4b46-80b4-4b098dab227d","remote_addr":"[fd00:1122:3344:101::a]:60476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44947,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:37.788155101Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7a62bc85-f839-4884-80ff-5450a61b33f4","remote_addr":"[fd00:1122:3344:101::a]:60476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:44.546140597Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54293"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:44.546284199Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"b811ae49-0624-4f40-bf3c-ed45b027241a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54293","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":118,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:11:49.648502553Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"deafcc0d-ea90-41ec-9d0e-d7b9e6e51d51","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":538,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:05.867555834Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50278"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:05.867756757Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"80e411dd-3dc5-410e-9260-6b3fce0a65d1","remote_addr":"[fd00:1122:3344:101::b]:50278","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":175,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:06.109000733Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:53573"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:06.155270285Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8299debb-41a8-46e2-ae98-0f923d5e2ae0","remote_addr":"[fd00:1122:3344:101::b]:53573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46201,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:06.15613369Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"982ae0b2-7abd-4bfe-a6e6-7adc8b9360df","remote_addr":"[fd00:1122:3344:101::b]:53573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":693,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.372189802Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63886"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.372384455Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"7b344942-c627-4b51-8b7f-22ad901f1cc4","remote_addr":"[fd00:1122:3344:101::a]:63886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.453430445Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:46123"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.454448013Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b641d173-d7c6-4408-903e-0d7fb9780f22","remote_addr":"[fd00:1122:3344:101::c]:46123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":980,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.713037817Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.762344943Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"bd426a90-5e50-4eeb-b332-e4b432a85b51","remote_addr":"[fd00:1122:3344:101::c]:55340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49261,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.762472695Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53078"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.810310565Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9fbfd5a3-7013-401f-8b4f-475c616a4ad1","remote_addr":"[fd00:1122:3344:101::a]:53078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47768,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.810515248Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8340f085-6b2f-436f-ac8b-e2c77fdf5c00","remote_addr":"[fd00:1122:3344:101::c]:55340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:07.810655751Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f99a2dfa-81c9-4c71-bd8e-046339cf7e09","remote_addr":"[fd00:1122:3344:101::a]:53078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:14.578518256Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:14.579212519Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"8dceccbc-701a-45fd-96dc-c5a5c307792a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57252","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":667,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:19.648643582Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"ba3445a5-06cf-4b97-b176-b19adca5b935","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":526,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:35.651255935Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56949"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:35.651523399Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"162b81e7-b32d-4732-bdc6-58fa87165afc","remote_addr":"[fd00:1122:3344:101::b]:56949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":228,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:35.824437421Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40654"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:35.871491407Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"617e06ca-3301-4d34-9443-7ebe831b0a70","remote_addr":"[fd00:1122:3344:101::b]:40654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46995,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:35.871727781Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"75dd3905-29b4-43b6-9cfe-8ffc4cd40004","remote_addr":"[fd00:1122:3344:101::b]:40654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.379724259Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.379924013Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"038de337-e8e9-409e-bc4b-f9b2b6576d1b","remote_addr":"[fd00:1122:3344:101::c]:55174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.397534656Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41112"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.397664408Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d36ef07a-fd03-48a1-b314-01bd63b38317","remote_addr":"[fd00:1122:3344:101::a]:41112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.471111843Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36160"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.519073265Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ad0e43d0-62e4-4e58-8bb8-f744af120886","remote_addr":"[fd00:1122:3344:101::a]:36160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47913,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.519308219Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f04a5ff5-ef9b-4afc-adec-76d9e212b618","remote_addr":"[fd00:1122:3344:101::a]:36160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.526229832Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58680"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.57119188Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fdcd7951-50fe-445c-8073-6db72dba650e","remote_addr":"[fd00:1122:3344:101::c]:58680","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44879,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:37.571433425Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"026390a9-95cc-42cb-96b5-71d447c631e8","remote_addr":"[fd00:1122:3344:101::c]:58680","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:44.610696771Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57651"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:44.610859404Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ff1d30ab-01da-4aa7-8cf3-a03a0d885b0e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57651","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:12:49.647838162Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"8773db3f-1c4c-40ce-b256-cc21a4f2657f","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":320,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:01.906077668Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:05.897643945Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33069"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:05.897847178Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"6fce6c28-c97c-4a1c-be4f-60cd44ef339f","remote_addr":"[fd00:1122:3344:101::b]:33069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:06.300155285Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:06.347244841Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1386707b-cdcc-4721-b638-96e807e05540","remote_addr":"[fd00:1122:3344:101::b]:34855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47032,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:06.347469085Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5fa9c8f8-7001-46c0-9e38-4dda5153f3b7","remote_addr":"[fd00:1122:3344:101::b]:34855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.494848708Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.495046272Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0bf68275-73e2-4814-bde9-ca1a798b5550","remote_addr":"[fd00:1122:3344:101::a]:58603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":178,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.581210362Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61604"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.581353255Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"ae10cb87-e405-4139-ad23-e9130b2b3730","remote_addr":"[fd00:1122:3344:101::c]:61604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.732370778Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.776900739Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9f8f1a57-2b6b-4067-82d3-e82e387e4e1b","remote_addr":"[fd00:1122:3344:101::a]:39676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44475,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.777014691Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35321"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.78031584Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"eacbcb9c-3403-43f0-b26d-e966e62c92d3","remote_addr":"[fd00:1122:3344:101::a]:39676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3086,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.821834357Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0b5db3b3-2ae5-40ba-8a9f-a00c1abef569","remote_addr":"[fd00:1122:3344:101::c]:35321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44726,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:07.822193343Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"84fcd351-e016-4825-a835-a9ab949ae106","remote_addr":"[fd00:1122:3344:101::c]:35321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:14.642853073Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63856"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:14.642946705Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"0430b149-ef65-476c-bdf4-8844da700ac8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63856","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":80,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:19.694962915Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"1d3af2ee-bf7f-4cac-8edc-b5becdaba5be","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":535,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:35.64332936Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:35.643550964Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0306c0a7-9cfa-4c41-a508-85bade4d987b","remote_addr":"[fd00:1122:3344:101::b]:38173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":202,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:35.769131285Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64409"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:35.814516501Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"24e4081a-585a-4fcc-b493-732d6354b6f7","remote_addr":"[fd00:1122:3344:101::b]:64409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45315,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:35.814805576Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"630f4313-c441-4d08-86e3-a1ccbb8c10df","remote_addr":"[fd00:1122:3344:101::b]:64409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.462167381Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57401"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.462335935Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"dbe9f896-fe4f-464e-b553-36e6af9c06c2","remote_addr":"[fd00:1122:3344:101::c]:57401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.510656383Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.510785985Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"43ff05eb-eb83-4830-a4ff-85360772c1b6","remote_addr":"[fd00:1122:3344:101::a]:62109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.565598819Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60088"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.610202422Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e6be9dfd-3b8f-40d4-8a3f-3e5598642c44","remote_addr":"[fd00:1122:3344:101::c]:60088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44559,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.611340002Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"6d4235f0-3340-4f90-8443-c9e33b691cd6","remote_addr":"[fd00:1122:3344:101::c]:60088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":977,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.633512065Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36796"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.678293871Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ade5e735-056c-4f3a-ba79-e49e622d44bb","remote_addr":"[fd00:1122:3344:101::a]:36796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44690,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:37.678572546Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a7de325a-9eb5-47f9-8fd3-1d87c6f26474","remote_addr":"[fd00:1122:3344:101::a]:36796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:44.674794925Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:44.674886347Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1cd4163c-378b-4d9a-8a52-be22a6bfee75","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64208","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:13:49.69222269Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"342cb8e1-be62-4de9-af03-6870cad5d439","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":539,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:05.89667818Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:37045"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:05.897716828Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0f6562d1-6553-4ada-a2ea-f627bb6a1063","remote_addr":"[fd00:1122:3344:101::b]:37045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":978,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:06.056333056Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63423"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:06.102584658Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c0fad7c1-7b3e-4e71-8088-86f149fdeca3","remote_addr":"[fd00:1122:3344:101::b]:63423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46215,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:06.102834432Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"862c901a-bcaa-44cb-b748-9c613dce377c","remote_addr":"[fd00:1122:3344:101::b]:63423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.382499694Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.382706717Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"c2a81112-46e6-4a04-ade9-e45301f6e319","remote_addr":"[fd00:1122:3344:101::a]:37480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.483314954Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33718"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.483494268Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"2004852c-cb46-4838-af8b-a29a819d47bc","remote_addr":"[fd00:1122:3344:101::c]:33718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":122,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.63845639Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47962"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.686250359Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5605d5c8-c6aa-4d9f-b4e2-21ba3bdd88db","remote_addr":"[fd00:1122:3344:101::a]:47962","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47731,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.686505464Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"49e50be0-4b02-46cf-b057-340f8c889677","remote_addr":"[fd00:1122:3344:101::a]:47962","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.814848274Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.860954063Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d8a8b997-a64d-4231-8e75-9db4562f7056","remote_addr":"[fd00:1122:3344:101::c]:37120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46028,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:07.861843909Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"013973cb-750e-4fa8-845b-79688353ad21","remote_addr":"[fd00:1122:3344:101::c]:37120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":735,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:14.708024704Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51148"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:14.708133836Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"c778023e-5dba-453f-a8d7-693ba1d353da","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51148","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:19.692443868Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"46f1a6dc-61ad-40fc-b007-048c1b7c2bf0","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":586,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:35.704596847Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44763"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:35.704812291Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"19468a5e-7d16-4d0b-bc54-47fc1f4b261f","remote_addr":"[fd00:1122:3344:101::b]:44763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":183,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:35.795452701Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:54215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:35.843908152Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9162f624-4260-4cf2-b375-dec1b135aa8f","remote_addr":"[fd00:1122:3344:101::b]:54215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48396,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:35.844204827Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"383cfe3c-da6d-4d57-be12-6f87b1b290db","remote_addr":"[fd00:1122:3344:101::b]:54215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.421132209Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63773"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.421889963Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"a5ee7783-f9ef-42c9-8efe-e5c7da8ee083","remote_addr":"[fd00:1122:3344:101::a]:63773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":746,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.425266333Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56946"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.425403025Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"ce4d931d-c6d0-408a-86d2-aedadd25837c","remote_addr":"[fd00:1122:3344:101::c]:56946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.510525007Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34395"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.559684111Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"faaf7fe4-c387-4956-9954-b3eed1ea7ca0","remote_addr":"[fd00:1122:3344:101::a]:34395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49115,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.560033787Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"756291cb-a32d-4dfa-a01e-43ed96732751","remote_addr":"[fd00:1122:3344:101::a]:34395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.597175597Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59518"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.641887531Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0d849a2c-c302-41ac-92f1-92ed8b1ac9bd","remote_addr":"[fd00:1122:3344:101::c]:59518","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44660,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:37.642183446Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3757cb0a-36ea-49fd-9c3b-c998ff4f9ede","remote_addr":"[fd00:1122:3344:101::c]:59518","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:44.743595876Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:44.743688347Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"3fe728f4-cc90-4e8d-a7ca-9054632b6bdb","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60384","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":80,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:14:49.692497877Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"ea787eec-d8ed-4ea7-8cc0-0d438d94a830","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":573,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:05.7363304Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:05.736606735Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"41462c5b-7592-44f3-813e-1f154941e326","remote_addr":"[fd00:1122:3344:101::b]:35346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":183,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.015985588Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63658"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.016203262Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.016253353Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.016284033Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.016300174Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.016311754Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.016344294Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.016352225Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.016368595Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"10bc1873-f7e2-426b-9833-4ec04b2d9350","remote_addr":"[fd00:1122:3344:101::b]:63658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":334,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.106293782Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.152039245Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2f3a3929-87f0-42f8-9f17-50bc27767aa2","remote_addr":"[fd00:1122:3344:101::b]:44266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45691,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:06.152990792Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"611315e4-5cc2-4aee-82aa-caba96d2cb34","remote_addr":"[fd00:1122:3344:101::b]:44266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":668,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.415650261Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.415878655Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"e4a85732-e5ae-4ab8-83fa-b43cee32f22a","remote_addr":"[fd00:1122:3344:101::a]:39396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":196,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.46064399Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51612"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.460806223Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d0d7b262-1bae-42ee-9aed-5a4f62114c28","remote_addr":"[fd00:1122:3344:101::c]:51612","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.521733666Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60463"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.52194837Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.52199068Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.522025111Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.522051511Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.522086162Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.522107352Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.522125813Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.522134893Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6aaf3d8e-1f8a-4a69-9ade-e55f66dcb52b","remote_addr":"[fd00:1122:3344:101::a]:60463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":355,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.592186487Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63255"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.5923516Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 13e81480-505c-4272-9ace-9773d023170c, kind: Service { id: 56096d63-69cd-4c67-9c8f-fe757a4566d3 }, name: Name(\"external-dns-56096d63-69cd-4c67-9c8f-fe757a4566d3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 13])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: f7c30c27-2106-484b-bc56-ac70941de2aa, kind: Service { id: 6d9390f7-43dc-4e13-9714-d2a358e2f5d4 }, name: Name(\"external-dns-6d9390f7-43dc-4e13-9714-d2a358e2f5d4\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 229, 173])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22e4185f-95a2-4f41-bd9f-bdf79a1f811c, kind: Service { id: d5da8fa4-2f8d-4c39-9334-8e19330e172f }, name: Name(\"nexus-d5da8fa4-2f8d-4c39-9334-8e19330e172f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 208])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9d3b17eb-c104-4cd7-b817-286398ef2b2e, kind: Service { id: c27f5e00-47bb-499a-ad9e-1146fabe9d48 }, name: Name(\"nexus-c27f5e00-47bb-499a-ad9e-1146fabe9d48\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 140])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: ad82ffd1-a739-4483-9b22-ad9556012366, kind: Service { id: 5cc3fd7a-881a-45bf-9524-016003ecf668 }, name: Name(\"nexus-5cc3fd7a-881a-45bf-9524-016003ecf668\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 105])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.592393561Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.592414511Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.592433662Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.592453482Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.592467082Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.592476532Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.592488713Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b8d82836-c28a-4622-9973-3dbd7462eb19","remote_addr":"[fd00:1122:3344:101::c]:63255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.730072017Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52874"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.775678347Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b15df732-7298-48ee-8259-dec8e84b572d","remote_addr":"[fd00:1122:3344:101::a]:52874","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45557,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.775923711Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3c4f85f6-5c37-4200-a905-0df7b66879d5","remote_addr":"[fd00:1122:3344:101::a]:52874","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.870006473Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.914617125Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"b11c6bf3-f284-4cd5-bcc3-3bef6a95b6e0","remote_addr":"[fd00:1122:3344:101::c]:48865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44521,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:07.914859809Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a8c90f02-faac-489c-b4dc-28b02b19b693","remote_addr":"[fd00:1122:3344:101::c]:48865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:14.776492429Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63277"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:14.776589591Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"148820bf-592c-4079-957a-b6e3621825ad","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63277","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":65,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:19.692821113Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"27a31d5b-2021-4dee-b4f9-be74b9863366","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":539,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:35.663877952Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:37833"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:35.664071156Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"651a7f58-ecaa-46c5-8b74-9a71e1248ced","remote_addr":"[fd00:1122:3344:101::b]:37833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":175,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:35.884322218Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46809"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:35.933331119Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e8f7592b-0707-4de8-af97-b1cefb60014a","remote_addr":"[fd00:1122:3344:101::b]:46809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48939,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:35.933614554Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d092ee25-5a1b-4e35-8417-209774d9fb7a","remote_addr":"[fd00:1122:3344:101::b]:46809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.337078165Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:38042"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.337304329Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d9db7095-3f46-466b-9ce9-23201406833d","remote_addr":"[fd00:1122:3344:101::a]:38042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":200,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.475907211Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53234"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.525049674Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5c444855-7fab-42ab-9334-4b6d82f3d1c8","remote_addr":"[fd00:1122:3344:101::a]:53234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49071,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.525200197Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:32960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.525466531Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"4bd3c5d2-7701-4e1d-8bdb-d49ef78d5b61","remote_addr":"[fd00:1122:3344:101::c]:32960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":216,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.525551523Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8710d425-0537-4212-996b-badbe86fd6ba","remote_addr":"[fd00:1122:3344:101::a]:53234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":215,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.599833173Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60819"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.644675359Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"29fb3132-8064-486c-9018-d302d7a55e76","remote_addr":"[fd00:1122:3344:101::c]:60819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44761,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:37.644958634Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8bae4cdf-f5a8-48e1-87c1-7dee7d037e42","remote_addr":"[fd00:1122:3344:101::c]:60819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:44.808494087Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:44.808612569Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"d93285da-72da-4e22-bce0-42cbf43a2f45","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40781","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":87,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:15:49.769211739Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"59ad9dd8-2e8e-4e98-8e63-3eb3b85b84b6","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":492,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:05.711996127Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65431"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:05.71220717Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"963b2601-e7d9-4b0f-b270-4ed40a6ddaf3","remote_addr":"[fd00:1122:3344:101::b]:65431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:05.978620563Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44748"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:06.024725802Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"410cd0d7-3194-4aaf-a51b-5c4f1cdb415b","remote_addr":"[fd00:1122:3344:101::b]:44748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46033,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:06.024972977Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a0dadc33-64f7-4480-8b2d-c3332215e092","remote_addr":"[fd00:1122:3344:101::b]:44748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.40496118Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64621"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.405145323Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"e1eae121-a01c-489f-bc4a-9c1b6df25ad6","remote_addr":"[fd00:1122:3344:101::a]:64621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":160,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.43029824Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.430525144Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"6b3ab661-15da-4802-906c-4037c82908d0","remote_addr":"[fd00:1122:3344:101::c]:61320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.619674585Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.666018668Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"103cd188-8c1b-4f90-981a-8429edfc29bf","remote_addr":"[fd00:1122:3344:101::c]:60307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46304,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.66614785Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:54133"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.669625142Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"aa17650e-53d2-4d7c-a356-bc2157a44c59","remote_addr":"[fd00:1122:3344:101::c]:60307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3257,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.708825558Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e548660a-ff91-431b-9dce-0c3f8fbebe25","remote_addr":"[fd00:1122:3344:101::a]:54133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42589,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:07.709189265Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3fc57444-a00e-4e95-8c96-5d8837de2723","remote_addr":"[fd00:1122:3344:101::a]:54133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:14.838952608Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36392"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:14.83909802Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"beb4d56f-8db4-45a6-b485-5ecaeee10db0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36392","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:19.769231259Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"696b4100-5a80-48c0-b895-29f7e639c2a9","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":561,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:27.854335063Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:27.854527166Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b1f5a4b3-a50e-444c-8981-1a61aa36bd53","remote_addr":"[fd00:1122:3344:101::a]:57883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":165,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:27.85531462Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(1275589) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:27.85534084Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"PUT","req_id":"4d3c6b34-7ecc-4864-a80a-3cea31581277","remote_addr":"[fd00:1122:3344:101::a]:57883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":629,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.017348049Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51723"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.021579134Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:546","metadata":"InstanceMetadata { silo_id: c44c3a56-1c6c-45e4-84cb-09f661fdfc45, project_id: c9ef865c-a9ee-41b3-aab7-502ed3b6e928 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-30T04:16:27.631784Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(fe68a66a-6283-428d-a0d8-78fc9912a771), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-30T04:16:27.659551102Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: fea462df-b254-4287-a947-a342cc88461f, kind: Instance { id: 3f458e42-852b-40df-ae34-e7c71d10bae3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1275589), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: fea462df-b254-4287-a947-a342cc88461f, kind: Instance { id: 3f458e42-852b-40df-ae34-e7c71d10bae3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1275589), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: fea462df-b254-4287-a947-a342cc88461f, kind: Instance { id: 3f458e42-852b-40df-ae34-e7c71d10bae3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1275589), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(1275589))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: fea462df-b254-4287-a947-a342cc88461f, kind: Instance { id: 3f458e42-852b-40df-ae34-e7c71d10bae3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1275589), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-53b30a23a56\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 78e1bdb7-6db9-4661-a894-5ed449ada38d, read_only_parent: Some(Volume { block_size: 512, id: 4f81eb9d-a7fd-4add-8460-f2dfd6f5e330, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 26c34ede-66b3-443b-8d4e-573690353318, key: Some(\"VGrFoM6ctTB5wVjzy7EXH1oMzOTJ7/e3OoGy3tAmO5M=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 78e1bdb7-6db9-4661-a894-5ed449ada38d, key: Some(\"ANvICRrbCFqVKPdepqMxy2h8Q+IARVsBojJy+qzXUUY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::18]:19003\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADGDSK+WL5YAAANIr5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAxg0ivli+WAAADSK+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIzZjQ1OGU0Mi04NTJiLTQwZGYtYWUzNC1lN2M3MWQxMGJhZTMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJT2FtamNNNjZCMDBZVnJLUnhGN2hYVzhjR1YzWTBqZm5GaElwcitnQkVBQyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.021671335Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:580","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.021810738Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: fea462df-b254-4287-a947-a342cc88461f, kind: Instance { id: 3f458e42-852b-40df-ae34-e7c71d10bae3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1275589), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: fea462df-b254-4287-a947-a342cc88461f, kind: Instance { id: 3f458e42-852b-40df-ae34-e7c71d10bae3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1275589), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: fea462df-b254-4287-a947-a342cc88461f, kind: Instance { id: 3f458e42-852b-40df-ae34-e7c71d10bae3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1275589), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(1275589))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: fea462df-b254-4287-a947-a342cc88461f, kind: Instance { id: 3f458e42-852b-40df-ae34-e7c71d10bae3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1275589), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-53b30a23a56\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 78e1bdb7-6db9-4661-a894-5ed449ada38d, read_only_parent: Some(Volume { block_size: 512, id: 4f81eb9d-a7fd-4add-8460-f2dfd6f5e330, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 26c34ede-66b3-443b-8d4e-573690353318, key: Some(\"VGrFoM6ctTB5wVjzy7EXH1oMzOTJ7/e3OoGy3tAmO5M=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 78e1bdb7-6db9-4661-a894-5ed449ada38d, key: Some(\"ANvICRrbCFqVKPdepqMxy2h8Q+IARVsBojJy+qzXUUY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::18]:19003\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADGDSK+WL5YAAANIr5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAxg0ivli+WAAADSK+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIzZjQ1OGU0Mi04NTJiLTQwZGYtYWUzNC1lN2M3MWQxMGJhZTMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJT2FtamNNNjZCMDBZVnJLUnhGN2hYVzhjR1YzWTBqZm5GaElwcitnQkVBQyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(fe68a66a-6283-428d-a0d8-78fc9912a771), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-30T04:16:27.659551102Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-30T04:16:27.631784Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.023717862Z","hostname":"sock","pid":1562,"uri":"/instances/3f458e42-852b-40df-ae34-e7c71d10bae3","method":"PUT","req_id":"a1938eba-1875-4ddf-bdf8-766d139dd173","remote_addr":"[fd00:1122:3344:101::a]:51723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6289,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.133017873Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41143"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.133209337Z","hostname":"sock","pid":1562,"uri":"/instances/3f458e42-852b-40df-ae34-e7c71d10bae3/state","method":"PUT","req_id":"1df2cce1-a94d-408c-96bb-76ffce937c64","remote_addr":"[fd00:1122:3344:101::a]:41143","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":125,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.156914168Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 250, 13, 97]), slot: 0, vni: Vni { inner: 1275589 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.157106901Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.19024072Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:28.210722024Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:29.327537562Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:29.327790247Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 49.469362ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:29.491640737Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 55.426653ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:29.549864412Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 179.118494ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:29.622351389Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 399.028039ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:29.81827196Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 923.064496ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:30.231434429Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 1.408708915s","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:31.166236465Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 781.081074ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:32.586120408Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 819.319639ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:33.376778983Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 777.090958ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:34.207010331Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.012217185Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"wait for service Some(\"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\") failed: Property not found. retry in 51.322104ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:35.020169926Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.082094156Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.1166356Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADGDSK+WL5YAAANIr5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAxg0ivli+WAAADSK+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIzZjQ1OGU0Mi04NTJiLTQwZGYtYWUzNC1lN2M3MWQxMGJhZTMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJT2FtamNNNjZCMDBZVnJLUnhGN2hYVzhjR1YzWTBqZm5GaElwcitnQkVBQyJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-53b30a23a56\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 78e1bdb7-6db9-4661-a894-5ed449ada38d, read_only_parent: Some(Volume { block_size: 512, id: 4f81eb9d-a7fd-4add-8460-f2dfd6f5e330, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 26c34ede-66b3-443b-8d4e-573690353318, key: Some(\"VGrFoM6ctTB5wVjzy7EXH1oMzOTJ7/e3OoGy3tAmO5M=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 78e1bdb7-6db9-4661-a894-5ed449ada38d, key: Some(\"ANvICRrbCFqVKPdepqMxy2h8Q+IARVsBojJy+qzXUUY=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::18]:19003\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 3f458e42-852b-40df-ae34-e7c71d10bae3, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: c9ef865c-a9ee-41b3-aab7-502ed3b6e928, silo_id: c44c3a56-1c6c-45e4-84cb-09f661fdfc45 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.117118098Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.238283781Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-30T04:16:35.238709898Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.238727139Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.238737379Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-30T04:16:35.238709898Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(fe68a66a-6283-428d-a0d8-78fc9912a771), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-30T04:16:27.659551102Z }","propolis_id":"fe68a66a-6283-428d-a0d8-78fc9912a771"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.23937927Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(fe68a66a-6283-428d-a0d8-78fc9912a771), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-30T04:16:27.659551102Z }, propolis_id: fe68a66a-6283-428d-a0d8-78fc9912a771, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-30T04:16:35.238709898Z } }","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-30T04:16:35.295656550Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.29567769Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.295704091Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-30T04:16:35.295656550Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(fe68a66a-6283-428d-a0d8-78fc9912a771), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-30T04:16:27.659551102Z }","propolis_id":"fe68a66a-6283-428d-a0d8-78fc9912a771"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.295713521Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(fe68a66a-6283-428d-a0d8-78fc9912a771), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-30T04:16:27.659551102Z }, propolis_id: fe68a66a-6283-428d-a0d8-78fc9912a771, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-30T04:16:35.295656550Z } }","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.679729113Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:61255"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.679864265Z","hostname":"sock","pid":1562,"uri":"/instances/3f458e42-852b-40df-ae34-e7c71d10bae3/state","method":"GET","req_id":"626a483e-bc03-4c35-801e-7f136fa83cee","remote_addr":"[fd00:1122:3344:101::b]:61255","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.735515594Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44329"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.736342689Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"33ef6407-2292-41a0-a65a-1404fb6c15a1","remote_addr":"[fd00:1122:3344:101::b]:44329","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":778,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.839759176Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.888094254Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"56b04e56-95e9-4973-9f20-d04d8c564006","remote_addr":"[fd00:1122:3344:101::b]:51142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48279,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:35.888348639Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"252b3600-d0c6-445f-9804-b95e19398fbc","remote_addr":"[fd00:1122:3344:101::b]:51142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.400424889Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58456"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.400618142Z","hostname":"sock","pid":1562,"uri":"/instances/3f458e42-852b-40df-ae34-e7c71d10bae3/state","method":"GET","req_id":"064eca28-f629-4d83-8937-6414d1750bf9","remote_addr":"[fd00:1122:3344:101::c]:58456","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.441091601Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60423"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.441304415Z","hostname":"sock","pid":1562,"uri":"/instances/3f458e42-852b-40df-ae34-e7c71d10bae3/state","method":"GET","req_id":"8a02b341-2456-4f98-89cd-3725429ecc14","remote_addr":"[fd00:1122:3344:101::a]:60423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.468131422Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.468308785Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"9d9e3459-9ebb-4ac3-9c8f-1c0d350dc0f4","remote_addr":"[fd00:1122:3344:101::a]:51060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.520433341Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55431"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.521190124Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8bbefeac-3aa0-40df-903b-a33f3e414660","remote_addr":"[fd00:1122:3344:101::c]:55431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":732,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.595005936Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35464"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.641735466Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"dc409f8a-a94c-4e0c-b041-9a8322aa84ea","remote_addr":"[fd00:1122:3344:101::a]:35464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46678,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.641873418Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36626"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.645569364Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"bb98aa4f-66bf-4985-953b-3d82f91c513b","remote_addr":"[fd00:1122:3344:101::a]:35464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3474,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.687465438Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"48697c33-c268-4753-a2e9-59e83491ebca","remote_addr":"[fd00:1122:3344:101::c]:36626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45484,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:37.687828615Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f26a59da-7215-448d-8db9-5e30c325b84a","remote_addr":"[fd00:1122:3344:101::c]:36626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:44.870939745Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:44.871565996Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"73df1138-6d68-44d5-9d95-6f89df48066b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43547","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":612,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.186092714Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41653"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.187434587Z","hostname":"sock","pid":1562,"uri":"/instances/3f458e42-852b-40df-ae34-e7c71d10bae3/state","method":"PUT","req_id":"d4cd457c-dd67-4e55-b64f-1f608dc1692b","remote_addr":"[fd00:1122:3344:101::a]:41653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1266,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-30T04:16:45.187546840Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.18758398Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.187623191Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-30T04:16:45.187546840Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(fe68a66a-6283-428d-a0d8-78fc9912a771), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-30T04:16:27.659551102Z }","propolis_id":"fe68a66a-6283-428d-a0d8-78fc9912a771"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.187651221Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(fe68a66a-6283-428d-a0d8-78fc9912a771), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-30T04:16:27.659551102Z }, propolis_id: fe68a66a-6283-428d-a0d8-78fc9912a771, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-30T04:16:45.187546840Z } }","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-30T04:16:45.207036176Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.207055246Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.207071506Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-30T04:16:45.207036176Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-30T04:16:45.207036176Z }","propolis_id":"fe68a66a-6283-428d-a0d8-78fc9912a771"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.207090567Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.207098187Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.208251517Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.208291948Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/75663f09-f516-40fc-852b-77e4a30319f5/debug/bundle/zone\", \"/pool/int/804b9788-bd41-4d87-a91b-e568478fa644/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/3022fa0e-6566-4854-a1a9-1e0c5908d5b0/crypt/debug\", \"/pool/ext/1793c88e-7478-4741-8254-860fae35146b/crypt/debug\", \"/pool/ext/f61d361f-e4c0-48df-9191-ff705041ee76/crypt/debug\", \"/pool/ext/8ce3eaf9-2970-4cb9-a1f6-c0a6fa0594c7/crypt/debug\", \"/pool/ext/5d12239f-a80e-4f75-83d4-ebef24b6fea1/crypt/debug\", \"/pool/ext/a168fba6-15db-4743-814f-b76d3daf999f/crypt/debug\", \"/pool/ext/2efcf3a6-ef7f-48c2-8713-0dd19fe385d1/crypt/debug\", \"/pool/ext/dc70dd36-578e-4d7e-9ca2-11c36573efc3/crypt/debug\", \"/pool/ext/8f547a7a-6177-4ca0-82ce-6646cf05d8a6/crypt/debug\"] }","zone_name":"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:45.845950385Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771\", bundle_id: 39b87f80-587e-43bb-9089-b147f407c824 }, time_created: 2024-05-30T04:16:45.208435660Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_fe68a66a-6283-428d-a0d8-78fc9912a771","v":0,"name":"SledAgent","level":40,"time":"2024-05-30T04:16:45.846057167Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:47.605027844Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:47.630885043Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-30T04:16:45.207036176Z }, propolis_id: fe68a66a-6283-428d-a0d8-78fc9912a771, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-30T04:16:47.630813642Z } }","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:47.730579234Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60273"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:47.730849139Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"ba0c83dc-195e-49f3-aedc-f0e1876db14b","remote_addr":"[fd00:1122:3344:101::c]:60273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":206,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:47.731119913Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(1275589) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:47.731165064Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"PUT","req_id":"4a42fcdc-b2ec-4ae5-8b56-df619f36655c","remote_addr":"[fd00:1122:3344:101::c]:60273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":119,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:47.896123025Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-30T04:16:45.207036176Z }, propolis_id: fe68a66a-6283-428d-a0d8-78fc9912a771, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-30T04:16:47.630813642Z } }","instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:47.899946323Z","hostname":"sock","pid":1562,"instance_id":"3f458e42-852b-40df-ae34-e7c71d10bae3","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:48.093147355Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:51396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:48.093325798Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"666ca6c0-2103-43cf-a4d9-094de155a98d","remote_addr":"[fd00:1122:3344:101::a]:51396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":145,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:48.093549862Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 250, 13, 97])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(1275589) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:48.093578152Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"DELETE","req_id":"4f103250-5568-4789-b088-561f213b98ba","remote_addr":"[fd00:1122:3344:101::a]:51396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":79,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-30T04:16:49.769947281Z","hostname":"sock","pid":1562,"uri":"/dd908086-f372-434c-b66b-6e5770f567d8","method":"GET","req_id":"dc16fd0e-28a5-4bea-bb62-bae9be123512","remote_addr":"[fd00:1122:3344:101::d]:33087","local_addr":"[fd00:1122:3344:101::1]:39972","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"dd908086-f372-434c-b66b-6e5770f567d8","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":892,"response_code":"200"}