[ May 5 01:48:27 Enabled. ] [ May 5 01:48:27 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ May 5 01:48:27 Method "start" exited with status 0. ] [ May 5 01:48:27 Rereading configuration. ] [ May 5 01:48:27 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.040339158Z","hostname":"sock","pid":1590,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.0405219Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.095344192Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.095387932Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.211044046Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.220999614Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.22874643Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.824173268Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.959875419Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.959909079Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.95998249Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.960095511Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.960116881Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.960156022Z","hostname":"sock","pid":1590,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.960194482Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.963280653Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:27.963295253Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.963307603Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.963315303Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.963341313Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.963385484Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.963511355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.963525575Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.963564906Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.975521603Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:27.975556573Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.103404487Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.360264697Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.360326357Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.360336328Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.360357548Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.360368728Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.360379918Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.360408968Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.369406847Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.369437258Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"fd584376-2139-48aa-bdc0-23bc960f5024 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.482361784Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737380255Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737424016Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737471156Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737483866Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737493617Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737537997Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737554587Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737571347Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737636338Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737642608Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737667528Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737672188Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737678108Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737682238Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737700239Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737704728Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737710779Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737722599Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737729189Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737737039Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737755319Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737760449Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737773479Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737778879Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737797249Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73780254Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73780784Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73781296Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73783318Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73783833Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.7378499Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73785536Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73785984Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73788185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73788649Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73789163Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73790349Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.73790791Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737936181Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737947561Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.737954101Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.738020252Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.738039552Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.738047632Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.738059392Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.738066292Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.738083402Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.738092802Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.738106112Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.749552666Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.749587196Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99995591s","next_cleanup":"Instant { tv_sec: 2994, tv_nsec: 696447866 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.749595836Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.749651587Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.751632416Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.751722967Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752467274Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752560945Z","hostname":"sock","pid":1590,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752841268Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1002"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752848068Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752896829Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752933769Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752957639Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752962489Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1062"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.752984659Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.75299739Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.75301777Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.75302878Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1103"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.75303328Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.75304155Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753075Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753085331Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753460614Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753616806Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753621936Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1136"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753664826Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753674886Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753806138Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753935299Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.753978469Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 4115e24c-e8db-4625-994c-3c144db05bf3, rack_id: 65790f8d-3cb9-4324-9682-a18211fa9b99, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.75409133Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55316","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 4115e24c-e8db-4625-994c-3c144db05bf3, rack_id: 65790f8d-3cb9-4324-9682-a18211fa9b99, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754184842Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754195331Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754231752Z","hostname":"sock","pid":1590,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754256282Z","hostname":"sock","pid":1590,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754261782Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754267752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754272952Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754322103Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754328633Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754335143Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.754341713Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.756779687Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.764744856Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.774738755Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.785648882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.795103306Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.803773622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.813171155Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.821925291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.831012441Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.840059491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.840105841Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.840110301Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.840114721Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.840118921Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.840123611Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.8510847Z","hostname":"sock","pid":1590,"sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:352"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.85113383Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.85114616Z","hostname":"sock","pid":1590,"sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:364"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.851205071Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 73.116639ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.857553224Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f518e198-504c-470f-b095-ebb0aa692a7a\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f518e198-504c-470f-b095-ebb0aa692a7a\" }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.862355371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.864831295Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.87943744Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.898746311Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.898792661Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.903804111Z","hostname":"sock","pid":1590,"sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:379"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.903815821Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.903820431Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.931932829Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 174.861471ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.932466804Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6fa46ea0-cd42-4ec7-b111-65bc1a0b4049\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6fa46ea0-cd42-4ec7-b111-65bc1a0b4049\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:28.988225375Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.10806256Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 107.627857ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.108735657Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9aa35791-43e4-4eb8-ac7b-42dbf5755efa\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9aa35791-43e4-4eb8-ac7b-42dbf5755efa\" }"} {"msg":"Mounting oxi_76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.208221971Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.217520522Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 210.727789ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.218185909Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"15845a2f-5c77-4e42-91af-f1d247ebc3cb\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"15845a2f-5c77-4e42-91af-f1d247ebc3cb\" }"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.221602633Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.221722894Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.221754484Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.221784794Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.430078304Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 295.175658ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.430829471Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"748cea86-4eab-4914-afdc-d918b4047356\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"748cea86-4eab-4914-afdc-d918b4047356\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.726991629Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 278.256833ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.727755917Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"65b1c844-17a6-470f-8658-1840c3b0f4f8\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:48:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"65b1c844-17a6-470f-8658-1840c3b0f4f8\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.993305742Z","hostname":"sock","pid":1590,"sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.994042699Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.99408602Z","hostname":"sock","pid":1590,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:29.99416691Z","hostname":"sock","pid":1590,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:34981","producer_id":"4115e24c-e8db-4625-994c-3c144db05bf3","route":"//4115e24c-e8db-4625-994c-3c144db05bf3"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:30.000380462Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"365.127179ms"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.007359171Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.050515777Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.050565788Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.050635568Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.050644639Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.051429816Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.051446177Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.051478637Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.051492847Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.05178277Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.051973262Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.051992992Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55316","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.052052223Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.052062193Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.052093903Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.052101173Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.23382453Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.233933821Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.233953021Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.233965291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.233980381Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:30.238844469Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.265670325Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52184"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.265874627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.265974958Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"17a77f51-011a-4e3a-9d06-e5824912c26c","remote_addr":"[fd00:1122:3344:101::1]:52184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":192,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.294732422Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58392"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.294821303Z","hostname":"sock","pid":1590,"uri":"/sled-role","method":"GET","req_id":"24a19a94-3f5c-4328-834d-5ae188a3f5a5","remote_addr":"[fd00:1122:3344:101::1]:58392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":60,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.295480859Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.295502609Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.295874163Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.296123366Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: ffd2b91c-dfa0-44db-b615-cf75a07f683c, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool) }, OmicronPhysicalDiskConfig { id: f8484f99-8e6a-420b-b65e-e282551aa98f, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool) }, OmicronPhysicalDiskConfig { id: 3af63f2e-6974-4ced-b4f1-b41863a0667c, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool) }, OmicronPhysicalDiskConfig { id: d29defba-5dde-4775-894b-5328988f6c38, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: 6016711c-532d-4238-a301-8a9cba14887f (zpool) }, OmicronPhysicalDiskConfig { id: 552e4dcd-9827-4e5d-a38d-50d7b661961a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool) }, OmicronPhysicalDiskConfig { id: e5e6f497-5aa0-4db4-a74e-21aab22ca8d6, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool) }, OmicronPhysicalDiskConfig { id: 3c1b73e5-6140-4ceb-b8c7-d64aeb702aec, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool) }, OmicronPhysicalDiskConfig { id: eaf45b33-8062-4933-9c09-e35611230002, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool) }, OmicronPhysicalDiskConfig { id: 0ec2eb01-8dbd-49c0-becc-2bfd6656071a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.327538536Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:346"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.327702998Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42990"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: ffd2b91c-dfa0-44db-b615-cf75a07f683c, pool_id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: f8484f99-8e6a-420b-b65e-e282551aa98f, pool_id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 3af63f2e-6974-4ced-b4f1-b41863a0667c, pool_id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: d29defba-5dde-4775-894b-5328988f6c38, pool_id: 6016711c-532d-4238-a301-8a9cba14887f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 552e4dcd-9827-4e5d-a38d-50d7b661961a, pool_id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: e5e6f497-5aa0-4db4-a74e-21aab22ca8d6, pool_id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 3c1b73e5-6140-4ceb-b8c7-d64aeb702aec, pool_id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: eaf45b33-8062-4933-9c09-e35611230002, pool_id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: 0ec2eb01-8dbd-49c0-becc-2bfd6656071a, pool_id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.328032391Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.328037801Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.328043231Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.328047561Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.328071811Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.328077382Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.328081362Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.328088462Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:30.413994731Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"608.529671ms"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.42093575Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:30.42094234Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:31.024651598Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"554.850261ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:31.185548109Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:31.380494767Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:31.54673172Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:31.546823381Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:31.546830561Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:31.546837891Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:31.546844281Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:31.546996653Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:31.547003273Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:31.773636374Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.652142507s"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:32.131761174Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:32.314337959Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:32.31450618Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:32.503313827Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_8d354229-67d2-4778-8fa1-24cf4429afe5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:32.702029762Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 65.075137ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:32.904272511Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:33.508198951Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.750874927s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:33.508259272Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 133.620867ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:33.508585425Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 161.41038ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:34.269093184Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:34.514989625Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:34.515188157Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 525.96466ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:34.72782759Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:35.865882971Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 855.436877ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:35.866380606Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_8d354229-67d2-4778-8fa1-24cf4429afe5","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:35.884400334Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:36.095239138Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:36.100981005Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:36.101010705Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:36.101041416Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:36.101047126Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:36.101051116Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:36.324051931Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:36.324079851Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:37.475449084Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:37.722133912Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:37.722187893Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:37.722195993Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:37.722214093Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:37.722226713Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:37.722370475Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:37.722382465Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:38.408780191Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:38.409073714Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:39.103115036Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:39.606911516Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"7.383664649s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:39.606968177Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:39.607792535Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:40.348327866Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.59447758Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:40.60969791Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.733461763Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.740564794Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.740609544Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.740617884Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.740627145Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.740649815Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.750771915Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.750801485Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.868549169Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.876382707Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.876411917Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.876421387Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.876433877Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.876444297Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.876575658Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.876589039Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.922115669Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_857bd84b-5498-49a6-9512-c8f97ed577a9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:40.930811735Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_857bd84b-5498-49a6-9512-c8f97ed577a9","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.084521814Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.092901356Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.092933837Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.092942457Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.092947747Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.092954307Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.102632843Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.102658503Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"6016711c-532d-4238-a301-8a9cba14887f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.217524328Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.226353256Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.226389956Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.226404626Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.226448837Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.226476297Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.226562658Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.226570878Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.269753875Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6016711c-532d-4238-a301-8a9cba14887f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.277710254Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_6016711c-532d-4238-a301-8a9cba14887f","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.431003139Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.439128739Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.43916662Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.43918187Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.43918722Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.43919327Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.448802485Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.448837235Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.570420668Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.578450857Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.578482407Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.578491568Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.578543098Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.578554458Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.57869529Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.57871751Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:41.611065859Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.624907096Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f5178f34-f85d-4d2a-9f4f-5d19e04fae27/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.632724124Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.678720108Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Looking for unencrypted datasets in oxp_f5178f34-f85d-4d2a-9f4f-5d19e04fae27","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.791698095Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.800521932Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.800569533Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.800583823Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.800588603Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.800593833Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.810773064Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.810802094Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.931155294Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.939405976Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.939461326Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.939469286Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.939488217Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.939497107Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.939643358Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.939651688Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.985090636Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d0d8d3b6-4496-4772-9553-97dafea7d189/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:41.993222667Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:42.094179355Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_d0d8d3b6-4496-4772-9553-97dafea7d189","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.150158928Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.15836166Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.15842438Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.15843085Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.1584359Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.158440621Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.168595601Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.168622801Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"16235c83-9817-4220-9cf4-0c733b95b0bd (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.296591556Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.305234082Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.305268712Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.305315642Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.305338523Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.305347993Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.305508295Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.305521195Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.355999224Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_16235c83-9817-4220-9cf4-0c733b95b0bd/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.364334756Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_16235c83-9817-4220-9cf4-0c733b95b0bd","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.517283938Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.52557065Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.525619471Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.525630251Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.525638851Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.525644211Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.535523538Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.535550289Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"22f621aa-0a05-43b7-9d04-54301aba846e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.651590986Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.659623036Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.659667516Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.659675666Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.659686636Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.659698996Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.659875928Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.659885698Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.708132205Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:42.712277476Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_22f621aa-0a05-43b7-9d04-54301aba846e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.716206155Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_22f621aa-0a05-43b7-9d04-54301aba846e","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.870024086Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.877902744Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.877957244Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.877976674Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.877984644Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.877991104Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.887981513Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:42.888019804Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"b038acd0-b651-49ff-832f-7b28f7bae91f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.024628353Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.033594902Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.033640593Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.033649323Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.033656763Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.033663743Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.033799484Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.033808824Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.07986589Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b038acd0-b651-49ff-832f-7b28f7bae91f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.088082441Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_b038acd0-b651-49ff-832f-7b28f7bae91f","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.233385408Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.240291346Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.240327126Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.240372017Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.240506218Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: fd584376-2139-48aa-bdc0-23bc960f5024 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.24076773Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.241205775Z","hostname":"sock","pid":1590,"uri":"/omicron-physical-disks","method":"PUT","req_id":"b09c8b74-017b-4ad1-ae50-f8259e3309c6","remote_addr":"[fd00:1122:3344:101::1]:42990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12913382,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.271921518Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.27212986Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42004"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.272433843Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.272438794Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.334904771Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.334943581Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.396182527Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.396224267Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.462755585Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.462794776Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.462807516Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.462853956Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.504854351Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.504889502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.504895092Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.558974307Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.563007086Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.567023136Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.579878503Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.584964624Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:43.589267366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:48:43.715183271Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:44.094542921Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Profile for oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:45.122913527Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:45.123088389Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:45.123366402Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:45.607244226Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:46.086843026Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:46.086947468Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:46.087009048Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc\") failed: Property not found. retry in 45.411452ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.095165439Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:46.095197909Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:46.098468941Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc\") failed: Property not found. retry in 60.374808ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.149149282Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc\") failed: Property not found. retry in 178.522384ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.219023604Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 32.137338ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.225515788Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 30.70357ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.231723539Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 115.002658ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.265545013Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 112.435693ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.273441911Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 155.949842ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.410169613Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc\") failed: Property not found. retry in 326.771753ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.427358593Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 206.839976ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.442820846Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 259.554814ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.58481287Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 491.671115ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.688067081Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc\") failed: Property not found. retry in 1.185265532s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.764130413Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 594.8269ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:46.853655128Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 1.005058419s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:47.187769031Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 1.224772145s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:47.458283526Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:47.728252645Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:47.728276615Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc\") failed: Property not found. retry in 1.252044196s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:47.959710613Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 525.649237ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:48.204718655Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe92:9b6c","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:48.290936187Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:48.296874136Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:48.416283607Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:48.658541852Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 871.363236ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:48.692372686Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 1.048415378s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:48.738228979Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:48.841901344Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:49.133693448Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"wait for service Some(\"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc\") failed: Property not found. retry in 1.084244156s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:49.219619478Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:49.330647816Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:49.496349304Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 670.722711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:49.572597808Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:49.696422262Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 635.796333ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:49.795386381Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:49.867708586Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:49.877042448Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:49.877090109Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:49.877100669Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8\") failed: Property not found. retry in 1.316426969s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:50.253490689Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc\") failed: Property not found. retry in 513.17622ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:50.31327791Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.428042745Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.43669029Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"wait for service Some(\"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6\") failed: Property not found. retry in 634.675471ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:50.440264256Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.573895977Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.673195699Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.695830512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.801859291Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.92620841Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.960108885Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.992074121Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.992166272Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.992594037Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:50.992607637Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080255543Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080362194Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080371804Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080380525Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080404145Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080412515Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080419625Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080427005Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080433535Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080439365Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080445985Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080453405Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080459045Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080467105Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080473975Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080480915Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080492666Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080496296Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080499986Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080504666Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080508486Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080512036Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.080515646Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.589823271Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_1730d30c-9024-41b1-8d54-8a7716afdadc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.589888802Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_35800f95-c945-4804-971d-3500ecfdd3a8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.589898612Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_2ae3c2fa-0a44-4ccb-8892-e311882b3ec6"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.589904272Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.590228615Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.590494998Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"53cf4e83-d088-4f80-9808-dfceb8b6e2a6","remote_addr":"[fd00:1122:3344:101::1]:42004","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8318289,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:51.592559708Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"failed to write DNS configuration (will retry in 139.869565ms)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:51.883213672Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:51.995476401Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"no record found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"20.062526886s"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.044702428Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.094536551Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.142974229Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:583"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.142984259Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.174021216Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.174890925Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.174896175Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.203059393Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.203145524Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45388"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.203337396Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.203345276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.225101351Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.225129542Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.243244121Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.243276411Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.260899045Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.260930536Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.261116687Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.261253919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.272563831Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.303522237Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.304164073Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.304171863Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.347703823Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 213, 89]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.347755014Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.34833498Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.34835532Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.348646843Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.349835815Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.372953483Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:52.391036072Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_47f74754-422f-46e5-ac13-472535be1254:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:53.002584467Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:53.002734299Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254\") failed: Property not found. retry in 50.78887ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:53.140734143Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254\") failed: Property not found. retry in 73.965019ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:53.199709646Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254\") failed: Property not found. retry in 129.311319ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:53.298048689Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254\") failed: Property not found. retry in 331.063258ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:53.440045793Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254\") failed: Property not found. retry in 966.709828ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:53.780539239Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:54.386145866Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:54.386328527Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"3561244d-a301-45f7-9ba1-3207c9c439b4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40233","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254\") failed: Property not found. retry in 594.510234ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:54.756887451Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254\") failed: Property not found. retry in 1.440795918s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:55.360986253Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254\") failed: Property not found. retry in 1.199638548s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:56.810902667Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.03307271Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_47f74754-422f-46e5-ac13-472535be1254"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.03312856Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.033456813Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.033748307Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"bdd330d5-c014-4acf-bee9-3c04fcc78707","remote_addr":"[fd00:1122:3344:101::1]:45388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":5830552,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.035923638Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:624"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.065036846Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.065204558Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:33838"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_47f74754-422f-46e5-ac13-472535be1254': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.077055235Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.077179146Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"f412bb4b-dd6e-4ebc-ad54-74ed70c8a4b5","remote_addr":"[fd00:1122:3344:101::1]:33838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":11844,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:48:58.078838352Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.190619317Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:48:58.190762199Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:62042"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_47f74754-422f-46e5-ac13-472535be1254': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:02.199920663Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:02.199997484Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"6433b90a-2e1d-4354-8b34-d2ebf1378b06","remote_addr":"[fd00:1122:3344:101::1]:62042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009126,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:02.203266347Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:02.399134573Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:02.399256754Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61217"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_47f74754-422f-46e5-ac13-472535be1254': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:02.408303664Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:02.408428165Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"78244b07-db04-498a-98fa-cc147d76d51f","remote_addr":"[fd00:1122:3344:101::1]:61217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9026,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:02.410369784Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:02.5424314Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:02.542530801Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50870"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_47f74754-422f-46e5-ac13-472535be1254': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.55110622Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.551203681Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"7e33f268-8d10-4f82-99f5-0774b79e85aa","remote_addr":"[fd00:1122:3344:101::1]:50870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008535,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:06.553440043Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.782127044Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.782256976Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:50974"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.791051863Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"ee32ba97-35f2-4753-af4b-516529af2e77","remote_addr":"[fd00:1122:3344:101::1]:50974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8636,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714873742.4234142, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.792880781Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:06.792888651Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.984161641Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.984256012Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.992521264Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"41e2eb87-d5c8-41e3-9436-e28086dc3740","remote_addr":"[fd00:1122:3344:101::1]:34406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8139,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714873742.4234142, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:06.993932708Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:06.993936958Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:07.364425671Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:07.364525192Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63845"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:07.373975345Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"7d3002af-3613-4129-be6c-2e47d49d9224","remote_addr":"[fd00:1122:3344:101::1]:63845","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9331,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714873742.4234142, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:07.37641397Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:07.37642227Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:07.713204299Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:07.71332499Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38623"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:07.722481201Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"a7535ece-5444-4b06-a740-75ece6bf994e","remote_addr":"[fd00:1122:3344:101::1]:38623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9064,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714873742.4234142, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:07.724751403Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:07.724755874Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.210476326Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.210569567Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58245"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.219121161Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"3eb81b8b-d249-4870-b9d1-8b31d6605d84","remote_addr":"[fd00:1122:3344:101::1]:58245","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8462,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714873742.4234142, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.221831278Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:08.221837908Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.703648112Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.703762283Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39354"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.711721301Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"5f43c0ce-57d3-400e-bf46-682498a04cb3","remote_addr":"[fd00:1122:3344:101::1]:39354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7889,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714873742.4234142, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.713298697Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:08.713304207Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.534473083Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.534634945Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55381"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.53819078Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.555559911Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"3285998e-d099-4f21-9978-969591328463","remote_addr":"[fd00:1122:3344:101::1]:55381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":20850,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2901090547, ip_addr: 172.235.32.243, stratum: 3, ref_time: 1714873747.5974438, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.557361699Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.557366689Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1215"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.586838581Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.586959172Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:40905"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.587325105Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.587338426Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.606798318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.606853649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.623322701Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.623359322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.641028816Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.641072657Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.693793028Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.693841849Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.742653901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.742714162Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.795765886Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.795812147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.845535298Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.845575319Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.894346411Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.894403592Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.894639234Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.894767165Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:08.926803462Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:08.926836742Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:08.926843552Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:08.926860433Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:08.926867162Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:08.926967243Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"14bc0c83-5ff0-4929-9242-88cca8a38d2f","remote_addr":"[fd00:1122:3344:101::1]:40905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\", TimeNotSynchronized), (\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\", TimeNotSynchronized), (\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\", TimeNotSynchronized), (\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\", TimeNotSynchronized), (\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\", TimeNotSynchronized)]","latency_us":339842,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:08.927193786Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"111.341403ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"14bc0c83-5ff0-4929-9242-88cca8a38d2f\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"14bc0c83-5ff0-4929-9242-88cca8a38d2f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.039343065Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.039682148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.039689208Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.06011853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.06014373Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.078922336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.078952126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.095867964Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.095894644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.114623549Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.114647579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.132590656Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.132615667Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.149432573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.149457043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.166834055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.166870895Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.185878693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.185908743Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.186051435Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.186186826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.21797128Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.21801152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.218018291Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.21802366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.218027971Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.218077281Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"21421390-ee56-40c1-a82c-39d2532685fb","remote_addr":"[fd00:1122:3344:101::1]:40905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\", TimeNotSynchronized), (\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\", TimeNotSynchronized), (\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\", TimeNotSynchronized), (\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\", TimeNotSynchronized), (\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\", TimeNotSynchronized)]","latency_us":178564,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.218214313Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"124.323801ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"21421390-ee56-40c1-a82c-39d2532685fb\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"21421390-ee56-40c1-a82c-39d2532685fb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.343539692Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.343728133Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.343735134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.360719821Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.360745331Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.377031503Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.377060053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.394475315Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.394504295Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.413118489Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.41315701Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.430990076Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.431015616Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.448676091Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.448707111Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.465966812Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.465991572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.483682977Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.483709537Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.483874179Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.48397686Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.514930756Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.514967676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.514975276Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.514980636Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.514984687Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.515037577Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"be26649d-da7d-4bf2-b1f5-3238f525a3b5","remote_addr":"[fd00:1122:3344:101::1]:40905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\", TimeNotSynchronized), (\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\", TimeNotSynchronized), (\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\", TimeNotSynchronized), (\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\", TimeNotSynchronized)]","latency_us":171412,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.515157368Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"211.195422ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be26649d-da7d-4bf2-b1f5-3238f525a3b5\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be26649d-da7d-4bf2-b1f5-3238f525a3b5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.727289146Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.727541598Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.727550198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.746345634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.746383384Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.763975858Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.764003218Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.783256419Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.783285899Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.802051505Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.802078235Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.819962792Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.819986152Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.838254783Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.838283973Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.855807516Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.855834666Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.873629762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.873662633Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.873790304Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.873878895Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.904446707Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.904473247Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.904480147Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.904486858Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.904493007Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:09.904559708Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f334b534-c883-4e3f-8c57-08161a2fba7e","remote_addr":"[fd00:1122:3344:101::1]:40905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\", TimeNotSynchronized), (\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\", TimeNotSynchronized), (\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\", TimeNotSynchronized), (\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\", TimeNotSynchronized), (\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\", TimeNotSynchronized)]","latency_us":177147,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:09.90470937Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"138.694534ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f334b534-c883-4e3f-8c57-08161a2fba7e\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f334b534-c883-4e3f-8c57-08161a2fba7e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.028809295Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.029120328Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.029136498Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.03140067Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.047651241Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.047674681Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.066011843Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.066038873Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.083638237Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.083676387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.101419993Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.101441823Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.119940316Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.119965866Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.13753811Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.13756366Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.155617709Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.155646019Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.173674536Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.173708447Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.173864368Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.173966569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.256283283Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.256333044Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.256338684Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.256343464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.256348184Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.347770817Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.351573795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.355429603Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.359774066Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.364271601Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.368983017Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.373435931Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.377615323Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.381716413Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:10.387885244Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.175302682Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.175428003Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.181563554Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.181688615Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.219166416Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.219386158Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.226069344Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.226178385Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.255926409Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:13.25602319Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 29.931506ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.324201994Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 34.900904ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.332087032Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 40.630079ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.360826496Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 38.025268ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.367624064Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 143.47909ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.374777004Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 58.278631ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.382722933Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 26.807692ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.402057424Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 86.932805ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.508083862Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 121.708291ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.805156069Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 262.39937ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:13.93673741Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 123.042536ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:14.073808295Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 204.971558ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:14.292058943Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 142.54778ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:14.459245305Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 62.610665ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:14.510846726Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 492.547203ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:15.378648575Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 548.385996ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:15.855375558Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 301.150195ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:15.947333267Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 408.046425ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.062107732Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 202.70546ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.084846337Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 1.19139552s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.094208749Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 1.045991806s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.126907283Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 1.101299001s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.258885407Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 295.529908ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.297393487Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 430.025369ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.482039023Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 438.436212ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.607721645Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 1.265591688s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:16.92175415Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 953.454447ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:17.057482172Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 1.391232208s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:17.183888291Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 1.187065552s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:17.295169991Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 906.690431ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:17.369726588Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 750.661077ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:18.021946847Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 1.055368056s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:18.19636767Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 1.465307679s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:18.286014097Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 591.434333ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:18.491975943Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 1.017611408s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:18.585195715Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 1.084498952s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:18.782494255Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 1.217478977s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:19.097665921Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 1.180771937s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:19.261072256Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 1.438983669s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:19.613050056Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 661.594759ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:19.7631282Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 1.402379546s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:19.877981825Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 679.600726ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:20.324384238Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 925.909293ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:20.434293975Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 557.104826ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:20.451782448Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 807.257462ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:21.013896665Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 1.126509529s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:21.02140852Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 914.034307ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:21.061459386Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 800.978883ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:21.288634681Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 833.223913ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:21.369095196Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 1.117797229s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:21.831585329Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f\") failed: Property not found. retry in 1.373141616s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:21.986244978Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 1.335954814s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:22.0986599Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 865.536996ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:22.157918445Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 1.086135674s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:22.211086421Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd\") failed: Property not found. retry in 1.221823521s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:22.960032276Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a\") failed: Property not found. retry in 1.241598436s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:23.033571663Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1\") failed: Property not found. retry in 930.359885ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:23.31054115Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 571.685735ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:23.445641396Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:23.619428774Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58282"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:23.619545296Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"5872468d-3ce7-46be-8453-1edb3e56e5de","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58282","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":60,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 800.940317ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:24.029394588Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 1.08650003s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:24.840575267Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 822.311367ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:25.93843106Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:26.293732013Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"224.298122ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 549.243124ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:26.775407895Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 858.438241ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:27.7054323Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 608.865363ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:28.577853884Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 770.336838ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:29.201243058Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 991.30144ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:29.980986137Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 563.738515ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:30.981862182Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 645.985669ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:31.556487483Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036\") failed: Property not found. retry in 1.359652249s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:32.212647059Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.594235488Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_79eb9c0b-678f-428e-87e8-b3971eea69cd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.594323589Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_5202a09b-51cd-458a-a4b5-1eba98659d6f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.594329869Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.594358649Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_fc28d1c0-a8d0-487d-831e-680914dfe67a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.594367699Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.59438403Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.595323539Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.599605771Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"11d9c546-1822-4831-9495-2cdcd32bde10","remote_addr":"[fd00:1122:3344:101::1]:40905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":23570650,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.632430676Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:60302"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.661069929Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"948e9b13-a41b-4db1-97a2-620b8e084448","remote_addr":"[fd00:1122:3344:101::1]:60302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28550,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:33.661300201Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:957","retry_after":"75.778022ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"948e9b13-a41b-4db1-97a2-620b8e084448\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"948e9b13-a41b-4db1-97a2-620b8e084448\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.762994226Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"e914d507-7a9d-4b0f-9971-ffa85aa7b2d6","remote_addr":"[fd00:1122:3344:101::1]:60302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":24491,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:33.763136058Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:957","retry_after":"125.389789ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e914d507-7a9d-4b0f-9971-ffa85aa7b2d6\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e914d507-7a9d-4b0f-9971-ffa85aa7b2d6\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:33.914148201Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"28fd8329-c3fc-45aa-b883-74e32560886b","remote_addr":"[fd00:1122:3344:101::1]:60302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService9/omicron6 in oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_0b7523a8-ade6-4ac8-a9c3-8fb502124036 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService9/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":24790,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:33.914306713Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:957","retry_after":"170.884259ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"28fd8329-c3fc-45aa-b883-74e32560886b\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"28fd8329-c3fc-45aa-b883-74e32560886b\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:34.114439461Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:38.445238558Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:38.448394049Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.843243084Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.843352585Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"ab2cec8a-6ff4-47c8-ae6d-f8f230ffb413","remote_addr":"[fd00:1122:3344:101::1]:60302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6214832,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.87712504Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.877340242Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53153"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.877639216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.877648576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.904521102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.904584112Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.929417949Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.929470959Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.954079013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.954128364Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.978151932Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:40.978204422Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.001238141Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.001284001Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.02738408Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.027449501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.05662119Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.05668016Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.084206963Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.084256714Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.165327048Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.165383478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.264867025Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.264924755Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.347560834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.347631045Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.421249815Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.421298495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.492249405Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.492292899Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.556416655Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.556471235Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.613304049Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.613365529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.670746788Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.670795358Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.728620083Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.728663392Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.786576135Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.786619816Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.845094236Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.845140146Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.905564225Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.905627696Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.905842548Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:41.905982519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.012891419Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01294169Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01294829Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01295371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.012958Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01296366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01296824Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.0129727Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01297727Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.0129817Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01298604Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.0129908Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01299579Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.013016631Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01302162Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01302584Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.01303002Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.013035071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.013039261Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.013136301Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"03937bc0-fb39-4561-af7e-492ae5c42fd5","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized)]","latency_us":1138819,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.013393074Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"114.574578ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"03937bc0-fb39-4561-af7e-492ae5c42fd5\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"03937bc0-fb39-4561-af7e-492ae5c42fd5\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.097350247Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"594.100011ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.129467855Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.129867269Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.129874629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.154949787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.155001398Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.177554082Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.177596532Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.197013935Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.197048365Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.217226955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.217267745Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.23894185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.23898364Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.258755547Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.258790967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.277591283Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.277624374Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.298086198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.298128367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.31758269Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.31761636Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.336256565Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.336311285Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.355327884Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.355365144Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.376357982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.376391043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.395825557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.395857616Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.415680292Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.415713152Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.437935353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.437967583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.457973171Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.458006942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.479188762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.479222962Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.501574444Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.501618944Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.523551752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.523598982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.544412176Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.544459529Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.544683731Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.544849503Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64541242Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64545308Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64545981Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64546458Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64546882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64547401Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64547845Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64548277Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645487161Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.64549147Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645504261Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645509061Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645514101Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645518291Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645522681Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645527211Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645532601Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645536741Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645541841Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.645639382Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c162377e-55ee-4292-b9b8-5ebf2eb9b2eb","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized)]","latency_us":517429,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:42.645820114Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"131.583868ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c162377e-55ee-4292-b9b8-5ebf2eb9b2eb\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c162377e-55ee-4292-b9b8-5ebf2eb9b2eb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.778353517Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.77865361Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.77866285Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.803171693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.803204333Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.827740276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.827771577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.849854296Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.849886786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.871069836Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.871099226Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.892129855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.892171745Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.913085573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.913121803Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.933888049Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.933922479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.95317034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.95319964Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.973001416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.973032227Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.993382209Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:42.993417529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.016019883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.016051573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.036186763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.036218003Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.055492764Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.055527685Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.073103619Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.073132009Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.093846245Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.093876435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.114727121Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.114758432Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.134959302Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.134989042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.15493845Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.154966231Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.174622035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.174656946Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.194465952Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.194500082Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.194695264Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.194806075Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292322512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292372687Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292379278Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292384668Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292391343Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292399403Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292405273Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292410743Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292417603Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292423673Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292446143Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292452094Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292457714Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292461734Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292467024Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292472744Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292477994Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292483544Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292489004Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.292616165Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"74efbd99-c0a8-4143-9db1-285984259035","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized)]","latency_us":515534,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:43.292802857Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"176.663467ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"74efbd99-c0a8-4143-9db1-285984259035\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"74efbd99-c0a8-4143-9db1-285984259035\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.470744051Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.471067674Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.471078224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.494608138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.494653538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.518512024Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.518551105Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.539327891Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.539371471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.557964606Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.558002766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.575776882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.575816533Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.593209795Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.593245056Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.611151273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.611187723Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.632827468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.632866628Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.653733675Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.653771065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.676728403Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.676768504Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.698847272Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.698892433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.720166634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.720209491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.740872199Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.740911859Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.762932878Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.762971748Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.783985576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.784025066Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.806110676Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.806155016Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.828077244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.828121064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.849974481Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.850017171Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.873735266Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.873805698Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.896029897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.896087748Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.89631066Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:43.896463171Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000031888Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000085609Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000094129Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000100319Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000105719Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000111189Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000116599Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000121919Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000127569Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000134559Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000150599Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000156049Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.00016137Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.00016807Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000173239Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.00018008Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.00018546Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.00019203Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.00019724Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.000295901Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"7ba20af2-aa67-451d-a347-c03d9fb720aa","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized)]","latency_us":530849,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.000466758Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"204.646003ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7ba20af2-aa67-451d-a347-c03d9fb720aa\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7ba20af2-aa67-451d-a347-c03d9fb720aa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.205348974Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.206214782Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.206227992Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.230404322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.230447493Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.251214078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.251251499Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.270316898Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.270359438Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.289962872Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.290002103Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.311260523Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.311312044Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.330837608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.330877848Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.350185969Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.35022242Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.371751263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.371792094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.392121345Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.392160936Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.412903891Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.412944892Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.432259157Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.432297043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.453500784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.453540894Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.475023787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.475058378Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.495223947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.495261988Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.516014213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.516048823Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.536204364Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.536249444Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.556211882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.556257072Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.575000718Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.575033369Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.593730324Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.593770664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.61353315Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.613582671Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.613778546Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.613913654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710582172Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710641732Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710650322Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710654942Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710662322Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710667942Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710675233Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710682923Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710688913Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710696263Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710722083Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710727873Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710733563Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710739243Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710745043Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710750583Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710756154Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710763363Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.710768674Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.710906085Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"6865b44e-ee3a-4b74-8c1b-071c928f0c0e","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized)]","latency_us":506779,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:44.711095337Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"119.308395ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6865b44e-ee3a-4b74-8c1b-071c928f0c0e\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6865b44e-ee3a-4b74-8c1b-071c928f0c0e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.832163117Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.833061126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.833068646Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.857230306Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.857284566Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.879485286Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.879527927Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.902671056Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.902718406Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.923309461Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.923352861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.945936575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.945983125Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.966748731Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.966789552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.987986462Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:44.988027642Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.008342094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.008385844Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.031414982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.031460863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.053689933Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.053733084Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.076272237Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.076322587Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.09875817Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.09880101Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.122893569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.12293839Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.146270661Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.146313021Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.169050257Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.169087927Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.19155952Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.19160717Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.213661559Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.2137003Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.236282383Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.236327824Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.259623005Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.259676495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.282545742Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.282595752Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.282814315Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.282957934Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382715395Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382780216Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382787286Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382794946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382802996Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382810686Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382816176Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382823706Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382827726Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382833616Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382844626Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382848726Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382856057Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382860086Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382867057Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382872737Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382876647Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382880427Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.382884447Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.382938757Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"0aeb6a38-215a-440c-a693-5171ba0be06a","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized)]","latency_us":552104,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:45.383137229Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"197.961639ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0aeb6a38-215a-440c-a693-5171ba0be06a\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0aeb6a38-215a-440c-a693-5171ba0be06a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.581808859Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.582248053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.582262673Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.605796507Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.605830387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.627418661Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.627448781Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.64953187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.649562061Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.671841471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.671871132Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.692762439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.692805719Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.714155501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.714183711Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.734555913Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.734584983Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.755318089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.755350329Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.781381407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.781414928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.803677228Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.803706069Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.825320823Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.825350493Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.847484043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.847511193Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.870157117Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.870189568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.891672051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.891705341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.913334705Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.913366136Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.936389594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.936420914Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.95818489Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.9582168Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.981162338Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:45.981197198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.00556322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.00560298Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.028997602Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.029038003Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.029221554Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.029352255Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.14275672Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.14281023Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.14281862Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.14282442Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.14283Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142837571Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142843341Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142850651Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142856551Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142862861Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142887851Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142895531Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142901501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142907781Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142915301Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142921012Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142928041Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142933761Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.142937972Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.143059613Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"9ad127f5-2636-4075-b856-d5fe674749e8","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized)]","latency_us":562581,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.143263065Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"235.682083ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9ad127f5-2636-4075-b856-d5fe674749e8\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9ad127f5-2636-4075-b856-d5fe674749e8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.379104403Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.379436716Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.379454076Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.40492443Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.404968669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.428754175Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.428797435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.450999526Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.451052216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.475916043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.475961903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.499452346Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.499501556Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.521925899Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.521966989Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.545971027Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.546007657Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.568705062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.568741263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.593394176Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.593433138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.618246894Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.618290574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.641676716Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.641726606Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.667602033Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.667640253Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.690730771Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.690768062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.713305585Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.713348176Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.736367854Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.736405624Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.759876487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.759922467Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.78232751Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.78236784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.804847863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.804886213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.828510537Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.828585358Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.850503545Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.850556196Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.850776238Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.850900889Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955452206Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955509646Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955522047Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955530307Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955535987Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955541897Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955545897Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955551817Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955556047Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955559957Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955573227Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955579387Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955586167Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955591957Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955597047Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955604577Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955609927Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955615147Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.955620537Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:46.955651058Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"26fd0475-7a5a-4343-a97a-2e68fe94e9c1","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized)]","latency_us":577968,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:46.9558383Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"341.058444ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26fd0475-7a5a-4343-a97a-2e68fe94e9c1\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"26fd0475-7a5a-4343-a97a-2e68fe94e9c1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.297198014Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.297534617Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.297550177Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.321911429Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.321971269Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.343490603Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.343549713Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.363790764Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.363842584Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.386984624Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.387034374Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.40873514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.40879734Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.430238693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.430288453Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.452336512Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.452382592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.47235654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.472406431Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.49556274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.495617161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.517005813Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.517055663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.537867174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.53791435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.558827468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.558878648Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.579514233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.579559143Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.602617562Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.60267699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.625260996Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.625317527Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.644541397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.644589468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.665140161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.665186162Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.684881487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.684951548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.705078806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.705132627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.724734551Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.724797082Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.724996074Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.725147396Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832461189Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.8325302Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.83253756Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.83254339Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.83255062Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.83255651Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.83256343Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.83256879Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832575921Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.83258193Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832607271Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832614391Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832618551Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832625401Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832631381Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832636891Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832642161Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832647491Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832654221Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:47.832789363Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"057fe72a-1a39-4396-8315-65192b53ad26","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized)]","latency_us":536898,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:47.832972334Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"525.965646ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"057fe72a-1a39-4396-8315-65192b53ad26\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"057fe72a-1a39-4396-8315-65192b53ad26\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.358865158Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.359238792Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.359259122Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.38329315Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.383349331Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.407665902Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.407720012Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.428893462Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.428943133Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.450308345Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.450368977Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.4731147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.473175391Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.494330161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.494378241Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.518181322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.518224868Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.541084474Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.541131677Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.563005352Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.563047744Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.585318963Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.585367333Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.60823843Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.608285421Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.627903425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.627951515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.650065855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.650109125Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.670220925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.670283735Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.692321583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.692375644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.711681115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.711724856Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.734117098Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.734170068Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.754173006Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.754224627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.776318646Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.776378826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.7979048Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.7979589Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.798152962Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.798311854Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902114763Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902159887Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902166284Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902170984Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902176284Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902180553Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902185794Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902190174Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902195444Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902199044Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902210814Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902214334Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902217574Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902220774Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902223984Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902227164Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902231294Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902235204Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902238414Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:48.902294705Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"bc359295-de34-4ada-8dbb-ebfde57a1454","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized)]","latency_us":544770,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:48.902490477Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"729.881378ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bc359295-de34-4ada-8dbb-ebfde57a1454\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bc359295-de34-4ada-8dbb-ebfde57a1454\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.63209762Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.632434313Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.632449053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.655858145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.655903326Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.678330368Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.678365388Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.698838981Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.698878402Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.721341525Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.721380195Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.742300542Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.742348473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.763313221Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.763347281Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.78344011Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.78347387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.804823562Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.804854282Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.82473713Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.82477616Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.845168522Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.845204432Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.865559874Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.865599084Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.889592393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.889627163Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.911871733Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.911908734Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.932987923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.933038313Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.954824249Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.954859529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.976003729Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.976038719Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.996508932Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:49.996542313Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.014219328Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.014251098Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.035306557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.035346077Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.057319195Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.057363936Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.057535497Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.057649888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159201905Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159259466Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159267766Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159273306Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159279876Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159286906Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159293066Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159300396Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159306086Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159333076Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159338767Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159344177Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159349587Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159355187Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159362887Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159366807Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159372067Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159377567Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159384677Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.159526878Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"15d0afce-7563-4353-a2d4-77764ebfa351","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized)]","latency_us":528708,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:50.159710597Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"620.09157ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"15d0afce-7563-4353-a2d4-77764ebfa351\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"15d0afce-7563-4353-a2d4-77764ebfa351\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.779701876Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.780051371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.78007146Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.803622573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.803671314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.828962594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.828997265Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.852461577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.852502488Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.873383385Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.873419545Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.892248502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.892300632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.913067148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.913108159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.93340504Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.93345536Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.95262608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.952662261Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.972664539Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.972701139Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.9914522Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:50.991492956Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.014916928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.014965948Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.039703754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.039754094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.06355459Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.063595651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.084935852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.084980912Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.107985631Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.108026531Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.129742656Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.129786387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.151093578Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.151132178Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.171876084Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.171920194Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.193971443Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.194025773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.215503166Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.215557677Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.215778689Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.215932111Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322057793Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322101923Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322109003Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322113833Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322119193Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322123863Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322127183Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322131314Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322134683Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322144834Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322148284Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322152504Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322156714Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322160854Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322169934Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322174334Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322177584Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322180714Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322184754Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:51.322219274Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5863ee71-b01b-4f9c-95c1-549280cc7bbf","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized)]","latency_us":543841,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:51.322393686Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.017565018s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5863ee71-b01b-4f9c-95c1-549280cc7bbf\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5863ee71-b01b-4f9c-95c1-549280cc7bbf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.338143025Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.339023594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.339036004Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.362471076Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.362499676Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.383747547Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.383772227Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.402794106Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.402821126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.422733853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.422758014Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.440828313Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.440862863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.460060374Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.460084154Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.479646028Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.479673898Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.500274232Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.500296982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.520163439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.52018521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.539858175Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.539881965Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.561273457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.561300377Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.581531438Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.581557288Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.601492046Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.601518206Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.620999809Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.62103699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.64023703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.64026069Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.659330109Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.659353079Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.678478988Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.678502768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.697084943Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.697109533Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.71600685Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.716037551Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.735865937Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.735899397Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.73611441Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.736240231Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.83196811Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.83201439Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.83202271Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.83202993Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.83203526Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.83204103Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832047801Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832053581Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832060701Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832087201Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832094121Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832099661Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832105431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832111111Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832116631Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832122021Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832127731Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832133091Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832140651Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:52.832251012Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d3a83b03-58f3-49b0-80e2-c6aad6c103fa","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized)]","latency_us":495289,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:52.832439194Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"510.484793ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d3a83b03-58f3-49b0-80e2-c6aad6c103fa\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d3a83b03-58f3-49b0-80e2-c6aad6c103fa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.342565672Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.342932635Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.342951886Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.366112415Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.366163536Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.387668929Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.387713299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.408455125Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.408494595Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.429022119Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.429064549Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.448115148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.448151108Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.467071566Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.467114786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.486974633Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.487020134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.507580747Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.50762125Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.528349353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.528389724Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.547494693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.547531996Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.567586482Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.567629763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.586341758Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.586389779Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.606089414Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.606129175Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.624524037Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.624569457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.644269543Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.644311583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.664279921Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.664320521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.684357379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.68440198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.703672661Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.703712741Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.722852861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.722911631Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.742426765Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.742485806Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.742744808Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.742878439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841596638Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841648449Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841657319Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841663289Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841670119Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841675449Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841682209Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841687729Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841695029Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.84171262Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841716661Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841721986Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841725736Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841729186Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841732673Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841736299Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.84174175Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.841747039Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.84175277Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:53.84181387Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"3f92b536-bab8-446c-9baf-6700323ca01d","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized)]","latency_us":500443,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:53.842007252Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.086156048s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f92b536-bab8-446c-9baf-6700323ca01d\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f92b536-bab8-446c-9baf-6700323ca01d\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.157849078Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34437"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.157924294Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b6424acc-ef76-43e5-a1b7-e96104b630db","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34437","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.925839296Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.926188089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.92620004Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.94939721Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.94945141Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.970355147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.970400068Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.9887366Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:54.98878234Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.008140082Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.008178612Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.028403373Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.028440693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.047555893Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.047603663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.066957775Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.067001205Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.087304757Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.087349433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.107190434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.107229064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.126289993Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.126335934Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.146560064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.146603395Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.165032197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.165072468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.184684932Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.184741393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.20363628Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.20367337Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.222874001Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.222913641Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.243132302Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.243169502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.261205291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.261241341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.279501472Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.279546663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.298032166Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.298072056Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.316227576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.316281757Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.316478699Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.31661855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407825214Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407876375Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407885065Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407890555Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407896155Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407903365Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407926795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407932435Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407939655Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407946536Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407952136Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407957725Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407964896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407971566Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407976676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407981796Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407988326Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.407993926Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.408000606Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:55.408100387Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"7774e6da-b07c-4c4c-acf2-b60fa590e0a4","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized)]","latency_us":483425,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:55.408313289Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"985.325704ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7774e6da-b07c-4c4c-acf2-b60fa590e0a4\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7774e6da-b07c-4c4c-acf2-b60fa590e0a4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.392607006Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.392931629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.3929412Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.415762516Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.415810786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.43534095Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.43538029Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.455215487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.455262577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.475488188Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.475537248Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.495260804Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.495301574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.51397228Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.51401352Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.533914597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.533960278Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.553068337Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.553113977Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.572429979Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.572471839Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.592121304Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.592156715Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.612693558Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.612729119Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.631952579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.631993639Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.653355051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.653399402Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.672798763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.672834623Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.691142035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.691182285Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.708583158Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.708615798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.727322394Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.727369054Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.746873048Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.746920048Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.767843075Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.767885426Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.787398869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.7874617Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.787657032Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.787776883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884607773Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884657254Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884663923Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884671183Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884676854Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884682374Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884697434Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884701484Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884706774Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884712294Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884717864Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884726594Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884733644Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884740464Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884745784Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884752864Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884758155Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884763275Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.884768475Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:56.884885566Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d62b62b6-668d-4fe6-a7a4-ff5e5db629fc","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized)]","latency_us":493485,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:56.885120098Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.949259593s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d62b62b6-668d-4fe6-a7a4-ff5e5db629fc\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d62b62b6-668d-4fe6-a7a4-ff5e5db629fc\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:57.685001437Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"1.130786381s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.843880906Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.844155189Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.844162959Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.867144676Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.867185067Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.887867412Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.887906582Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.908052882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.908108693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.92700966Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.927068431Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.948263591Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.948301231Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.967036897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.967078607Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.985130636Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:58.985170987Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.003135835Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.003170875Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.024379855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.024422446Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.042437914Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.042494895Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.06119121Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.061228561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.080663513Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.080701034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.098791263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.098824823Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.1186745Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.118724231Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.138515667Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.138549557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.15696523Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.15700433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.175981458Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.176021799Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.19434042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.194381021Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.212295608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.212337889Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.231113915Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.231149025Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.231318447Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.231437008Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325356309Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.32541385Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.32542352Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.32545155Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.32545994Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.32546639Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.32547289Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.32548134Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.32548962Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325495931Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325503371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325509881Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325518161Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325526481Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325534371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325539161Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325545331Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325550311Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325554841Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:49:59.325639092Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a0f6ef25-f8ab-4153-8762-fcde868636b5","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized)]","latency_us":482957,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:49:59.325815449Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.723066718s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0f6ef25-f8ab-4153-8762-fcde868636b5\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:49:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a0f6ef25-f8ab-4153-8762-fcde868636b5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.043264141Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.044622534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.044630355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.067916216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.067953336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.08951985Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.08955585Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.111860371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.111908512Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.131123182Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.131164012Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.150635165Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.150680416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.169016428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.169061148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.187240619Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.187289439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.206798362Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.206844913Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.226210705Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.226260975Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.245314394Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.245358704Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.263552535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.263600516Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.281475813Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.281512593Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.299576222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.299616502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.317945914Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.317996155Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.336045194Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.336087504Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.354933771Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.354986981Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.372796398Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.372857699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.391299931Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.391345462Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.409224479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.409263689Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.427651802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.427704252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.427935625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.428118826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520637124Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520717754Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520726845Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520742895Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520749115Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520757215Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520770215Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520776445Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520783865Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520790995Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520795335Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520801155Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520805385Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520814475Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520818825Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520824835Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520841236Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520846706Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.520850686Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:02.520894116Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"cdc73266-85dd-4a8d-afb0-8cded3d98cb3","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized)]","latency_us":478661,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:02.521080468Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.657439365s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cdc73266-85dd-4a8d-afb0-8cded3d98cb3\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cdc73266-85dd-4a8d-afb0-8cded3d98cb3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.172881604Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.173698653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.173712263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.195952953Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.196010434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.215247795Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.215317035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.231531336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.231605187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.249616375Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.249668006Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.264924007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.264979768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.280135888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.280182078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.298739202Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.298789363Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.316093364Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.316141185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.333666758Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.333713819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.352400814Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.352435155Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.369477604Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.369515274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.386348041Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.386394576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.403800994Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.403846664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.422501809Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.4225467Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.440960692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.441012443Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.45891132Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.458955661Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.478455754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.480749317Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.497485543Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.497548233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.516057727Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.516112177Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.532879504Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.532928274Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.533090036Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.533202017Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.625961316Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626017707Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626030937Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626052347Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626058007Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626062077Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626066117Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626070167Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626075548Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626080848Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626095128Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626100718Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626110258Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626116218Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626121918Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626127268Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626132688Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626138278Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.626145518Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:05.626151938Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8e1efe53-6426-40d7-aa65-610fbffcdb14","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized)]","latency_us":454382,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:05.62636378Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.400501227s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e1efe53-6426-40d7-aa65-610fbffcdb14\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e1efe53-6426-40d7-aa65-610fbffcdb14\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.018767859Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.019590768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.019602738Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.04299784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.04305623Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.063672725Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.063720235Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.082653943Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.082707063Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.101758092Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.101809223Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.119919102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.119962643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.138106663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.138146253Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.156451864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.156497455Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.175281011Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.175325532Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.193178569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.193224279Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.211594091Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.211644112Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.22960081Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.22964051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.248212664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.250523617Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.266305653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.266347314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.285496134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.285539614Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.303499302Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.303552463Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.321985115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.322031046Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.341051484Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.341104945Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.35874816Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.35878621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.37689589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.37694549Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.395015909Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.39507316Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.395279252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.395431834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492068641Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492142132Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492152022Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492173182Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492179142Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492190613Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492197263Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492205379Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492211501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492230663Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492235263Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492240863Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492246673Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492251893Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492257253Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492262614Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492266694Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492271043Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492286594Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:09.492291414Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8b208f43-761c-4542-9c2f-8ffdbc136a9a","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized)]","latency_us":474697,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:09.492467136Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.243215316s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8b208f43-761c-4542-9c2f-8ffdbc136a9a\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8b208f43-761c-4542-9c2f-8ffdbc136a9a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.725049013Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.725655699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.725669659Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.748105911Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.748157612Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.768651295Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.768709206Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.7893549Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.789416991Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:13.806863934Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"2.060416603s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.808116336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.808124926Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.829217115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.829264256Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.847263744Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.847319225Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.864636237Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.864682637Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.882005398Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.882055499Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.900526732Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.900586463Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.918180287Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.918234888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.936239006Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.936285197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.953932442Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.953978352Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.972037514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.972087692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.989434974Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:13.989478464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.007601674Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.007657104Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.029843684Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.032079946Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.049094305Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.049141355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.06675741Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.066807542Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.084324314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.084370915Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.103361743Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.103410213Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.103608426Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.103734167Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194033202Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194144003Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194159743Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194168203Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194177643Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194189633Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194199733Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194209994Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194220074Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194239374Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194249334Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194259234Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194269354Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194282754Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194289505Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194295114Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194307705Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194315975Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194325015Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:14.194381135Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"75dfab5e-9378-4db0-8225-e2d222270bc7","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized)]","latency_us":470354,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:14.194598868Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.529525423s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75dfab5e-9378-4db0-8225-e2d222270bc7\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"75dfab5e-9378-4db0-8225-e2d222270bc7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.712822207Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.71312898Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.71313708Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.737835315Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.737890475Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.760160876Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.760217367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.779012293Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.779057963Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.798205553Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.798261774Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.817089571Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.817141111Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.834636825Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.834694325Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.853525852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.853587732Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.871927174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.871973615Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.89272083Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.892777741Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.910986681Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.911038222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.92898711Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.92902695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.946826417Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.946872567Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.965021007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.965070058Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.983030516Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.983079276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.999483649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:18.999527039Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.019098773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.019168384Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.037300745Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.037344834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.056981339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.057029389Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.076367161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.076415281Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.093893695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.093942385Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.094155541Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.094298879Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185295281Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185375922Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185387562Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185410412Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185419432Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185429242Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185443952Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185449392Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185455862Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185461022Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185466003Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185471112Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185479363Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185485943Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185499263Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185504573Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185526583Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185534893Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185540103Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:19.185546393Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"fb79c732-37d1-4158-b76d-e2199d262f70","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized)]","latency_us":473890,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:19.185754985Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.020087618s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb79c732-37d1-4158-b76d-e2199d262f70\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fb79c732-37d1-4158-b76d-e2199d262f70\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.196458475Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.197043791Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.197057251Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.217637284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.217687845Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.239613922Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.239677473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.258273898Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.258316258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.276858902Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.276903752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.295551857Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.295600737Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.311816888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.311874769Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.328958998Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.329004599Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.346534093Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.346589797Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.364540015Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.364585251Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.382402258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.382446278Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.401013662Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.401062703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.418595497Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.418640597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.436357589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.436397623Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.454126629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.454178289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.47340687Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.473447401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.492753582Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.492802623Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.51076115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.510819311Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.528351155Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.528388305Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.54601075Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.546058573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.564879397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.564945788Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.5651619Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.565296261Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.654936289Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.65500578Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.65501547Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.65503088Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.65503866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.65504607Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.65505521Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655064771Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.65507404Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655081591Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655088531Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655103421Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655111801Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655121011Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655126271Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655132851Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655143891Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655150551Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655163941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:23.655219652Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"20ba6b6d-6f27-4270-8727-12223b4a1169","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized)]","latency_us":459764,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:23.655419354Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.798028358s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20ba6b6d-6f27-4270-8727-12223b4a1169\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"20ba6b6d-6f27-4270-8727-12223b4a1169\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:24.109428855Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39882"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:24.109495956Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"1b23be6e-2acd-4139-add6-75961ab932fc","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39882","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.44715273Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.44821138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.44822675Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.470003796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.470073317Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.490683481Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.490732902Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.509581448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.509639259Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.528178043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.528225403Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.547482914Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.547537335Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.565956447Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.566030098Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.585273538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.585319639Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.602916213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.602959894Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.621673159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.621722674Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.64097185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.641017211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.658884018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.658934138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.67722373Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.67727536Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.696981855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.697022606Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.715170136Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.715214636Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.73374098Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.733807631Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.752347421Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.752393045Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.77201595Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.77205702Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.790425682Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.790474502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.810065567Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.810113097Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.827815593Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.827873063Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.828087575Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.828221397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919712754Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919784504Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919799475Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919806504Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919819235Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919826195Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919835695Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919844295Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919859105Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919868455Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919875109Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919883274Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919900712Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919905945Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919911086Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919916206Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919921106Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919928036Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.919934466Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:26.919998917Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"3f026eae-8639-4143-9b44-2c6e1fc9498e","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized)]","latency_us":473879,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:26.920233939Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"7.492210094s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f026eae-8639-4143-9b44-2c6e1fc9498e\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f026eae-8639-4143-9b44-2c6e1fc9498e\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:30.857360359Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"5.365521717s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.39319901Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.393544284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.393554454Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.416111768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.416164528Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.436711442Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.436761922Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.456326386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.456361727Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.475643318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.475693928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.494452214Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.494502815Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.512344062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.512384252Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.53133291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.53137905Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.550091386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.550133876Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.569100295Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.569144294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.587280224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.587322584Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.605932519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.605979739Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.623933947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.623976473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.642912717Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.642960766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.661448069Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.66150514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.679577579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.679633709Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.697967911Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.698017832Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.715499235Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.715550016Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.734441593Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.734489493Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.752846605Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.752903106Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.771961495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.772018425Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.772287388Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.772432169Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866316407Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866385471Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866394371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866402071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866416541Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866423911Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866430751Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866439351Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866446471Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866454761Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866475822Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866485352Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866491952Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866498592Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866506612Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866513612Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866520502Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866525162Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866531652Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:34.866637903Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"9746ae83-8365-4c66-818d-8a9b687bab51","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized)]","latency_us":474567,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:34.866885996Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"7.060056875s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9746ae83-8365-4c66-818d-8a9b687bab51\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9746ae83-8365-4c66-818d-8a9b687bab51\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.908820382Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.909146886Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.909157436Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.934944691Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.934998622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.960453664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.960511615Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.983678605Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:41.983728535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.006907724Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.006966065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.030155115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.032887812Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.054284825Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.054344375Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.076455764Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.076517355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.099590364Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.099643394Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.124054446Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.124113327Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.149448688Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.149510738Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.175417585Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.17547196Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.200165151Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.200228861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.223775605Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.223827875Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.247683742Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.247749603Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.27177125Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.271833021Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.296384825Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.296433465Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.322018529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.322070346Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.346616063Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.346670573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.370354798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.370421299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.39581632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.395884871Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.396060703Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.396205074Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514169534Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514222034Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514227724Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514233834Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514239484Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514243944Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514249354Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514253625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514259034Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514266045Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514284135Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514291625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514296455Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514300345Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514305625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514312295Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514316855Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514320385Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.514323875Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:42.514362766Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e71944f9-61b0-45c3-8945-8dafc7b53779","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized)]","latency_us":607052,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:42.515213014Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"11.959389287s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e71944f9-61b0-45c3-8945-8dafc7b53779\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e71944f9-61b0-45c3-8945-8dafc7b53779\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:51.203418442Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"9.423912544s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.060695235Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62433"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.060777096Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"81cb6e51-54ff-447b-bcbe-389d1c221032","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62433","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.443691792Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.444182617Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.444190207Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.466989693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.467057504Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.489321324Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.489371375Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.508400544Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.508445504Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.527538163Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.527591374Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.546070717Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.546125138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.565342868Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.565384529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.584131684Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.584175625Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.603005482Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.603068082Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.622635156Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.622683747Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.641837096Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.641883597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.660804544Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.660860205Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.679700972Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.679751342Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.698433257Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.698468108Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.717065052Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.717111523Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.736228562Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.736276873Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.754636265Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.754695235Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.773797935Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.773851435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.792298078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.792342659Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.810966553Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.811016724Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.82880649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.82886187Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.829197994Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.829334985Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921076545Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921140385Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921150576Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921157125Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921176946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921184056Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921194896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921203506Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921209676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921219546Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921228076Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921241596Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921246926Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921253957Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921258946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921263927Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921268777Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921275137Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921282057Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:50:54.921340127Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e2efbfa0-3a4e-4add-8d6b-46538345b0d8","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized)]","latency_us":478711,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:50:54.921556789Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"14.225053983s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e2efbfa0-3a4e-4add-8d6b-46538345b0d8\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:50:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e2efbfa0-3a4e-4add-8d6b-46538345b0d8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.109135052Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.109497266Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.109505546Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.132199821Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.132249281Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.149394711Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.149438332Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.16841785Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.16846638Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.186646721Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.186697151Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.207597208Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.207647319Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.225057601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.225111922Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.242446854Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.242496294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.260983828Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.261039618Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.278729233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.278788164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.297779942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.297820717Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.31366969Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.31371874Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.332563707Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.332621548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.350159412Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.350205532Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.367529334Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.367573524Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.386190439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.386252099Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.404853774Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.404897274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.422347937Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.422395368Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.440732059Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.44078078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.459332864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.459379073Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.477574575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.477640185Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.477903248Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.47807953Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569309234Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569392595Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569406255Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569411635Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569437795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569446835Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569462166Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569469286Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569477816Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569486436Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569494536Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569502496Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569516276Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569521376Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569533007Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569538276Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569547916Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569555007Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.569566687Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:09.569636757Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"41434a0e-4759-42e5-adf6-27779150e007","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized)]","latency_us":461596,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:09.5698721Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"12.268571215s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"41434a0e-4759-42e5-adf6-27779150e007\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:51:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"41434a0e-4759-42e5-adf6-27779150e007\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:15.597776705Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"19.913021743s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.806337711Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.806637473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.806645963Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.829566691Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.829618641Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.850403577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.850451718Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.869179433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.869218514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.886896629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.886936479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.904581493Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.904625285Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.923676514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.923723464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.942337649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.942392569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.960513079Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.960565479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.979291395Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.979332515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.998528246Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:21.998571906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.017538904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.017585174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.036557162Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.036611972Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.055531659Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.055573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.074244434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.074292695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.093099211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.093142871Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.110984958Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.111031498Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.130300089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.130350159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.152271846Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.152312736Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.170777839Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.170816379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.188864348Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.188916908Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.189170501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.189305612Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280526924Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280575014Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280599514Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280605795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280622405Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280628715Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280636395Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280642445Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280649855Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280655815Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280660035Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280665725Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280669715Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280673885Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280677835Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280681935Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280686015Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280710026Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280715746Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:22.280769066Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"27f7d637-d7d7-4f2a-b02b-f61680ce471f","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized)]","latency_us":474831,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:22.280962118Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"17.455007123s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"27f7d637-d7d7-4f2a-b02b-f61680ce471f\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:51:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"27f7d637-d7d7-4f2a-b02b-f61680ce471f\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:24.018732568Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58013"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:24.018868039Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"067771b9-34a2-4832-9806-83cec00c19b5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58013","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":66,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.736689287Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.737214212Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.737222402Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.761219399Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.76126601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.782330268Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.782377198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.802114904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.802166284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.821294473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.821338224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.839733456Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.839776076Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.85939613Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.859445691Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.87754609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.87759163Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.896459566Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.896508897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.915113521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.915160631Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.932712315Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.932758325Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.950675793Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.950733493Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.968186966Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.968229026Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.986689529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:39.986742969Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.004796028Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.004836608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.023458182Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.023501473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.042831024Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.042874124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.060144065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.060184875Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.077646758Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.077695138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.09605687Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.0960983Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.113896856Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.113957397Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.114203989Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.114344631Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204787365Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204854175Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204862536Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204868596Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204881316Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204893636Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204900986Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204906536Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204912176Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204919546Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204925106Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204932376Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204939456Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204943907Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204954667Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204958976Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204963207Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204967027Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.204971017Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:40.205055817Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"0878528b-cfaa-449b-a922-68f4ff5b43d1","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\", TimeNotSynchronized), (\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\", TimeNotSynchronized), (\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\", TimeNotSynchronized), (\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\", TimeNotSynchronized), (\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\", TimeNotSynchronized), (\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\", TimeNotSynchronized), (\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\", TimeNotSynchronized), (\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\", TimeNotSynchronized), (\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\", TimeNotSynchronized), (\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\", TimeNotSynchronized), (\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\", TimeNotSynchronized), (\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\", TimeNotSynchronized), (\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\", TimeNotSynchronized), (\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\", TimeNotSynchronized), (\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\", TimeNotSynchronized), (\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\", TimeNotSynchronized), (\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\", TimeNotSynchronized)]","latency_us":468138,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:40.205241979Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"21.679170816s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0878528b-cfaa-449b-a922-68f4ff5b43d1\", \"content-length\": \"124\", \"date\": \"Sun, 05 May 2024 01:51:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0878528b-cfaa-449b-a922-68f4ff5b43d1\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:51:50.530801029Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"30.938159042s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:54.050797098Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:51:54.050862169Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"4ca397e5-35e9-4378-ad43-5e48c5c25e7a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41491","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":40,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 35800f95-c945-4804-971d-3500ecfdd3a8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1730d30c-9024-41b1-8d54-8a7716afdadc, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 47f74754-422f-46e5-ac13-472535be1254, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 53823aaf-0b6e-4413-8f50-42f7413d72c2, kind: Service { id: 47f74754-422f-46e5-ac13-472535be1254 }, name: Name(\"ntp-47f74754-422f-46e5-ac13-472535be1254\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: fc28d1c0-a8d0-487d-831e-680914dfe67a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: 079c91fb-c032-47d7-8d99-720e56571e78, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd461cb3-4566-4efc-b780-d12158809749, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f4013866-e0b3-4697-9c6d-9fea86946018, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ecaf7873-a899-4ed4-9a5d-5fa055a6991d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 74918dc1-e131-42b3-a941-d060a1358fb7, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 36b5db01-3ec6-4b79-8cd0-3bc225315711, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bf77d87d-9513-4b4d-9600-4f57755c3178, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 9b3d43d8-586a-4880-98e4-d4c4925a2ac7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4fcc4205-2962-474d-86f6-0c103405365d, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External } } } }, OmicronZoneConfig { id: efd17334-7981-45c7-aa72-f1707204494e, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External } } } }, OmicronZoneConfig { id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3221346-1766-4a71-8cea-195049832d24, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External } } } }, OmicronZoneConfig { id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 89153b11-b7aa-463f-8e21-1efa1222641d, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External } } } }, OmicronZoneConfig { id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.885547863Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.885958807Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.885971297Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 35800f95-c945-4804-971d-3500ecfdd3a8, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.908036475Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.910054375Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1730d30c-9024-41b1-8d54-8a7716afdadc, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.926535148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.926587729Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ae3c2fa-0a44-4ccb-8892-e311882b3ec6, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.946324694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.946370554Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc28d1c0-a8d0-487d-831e-680914dfe67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.964762416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.964812347Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b7523a8-ade6-4ac8-a9c3-8fb502124036, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.984215759Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:01.984260479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 79eb9c0b-678f-428e-87e8-b3971eea69cd, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.003209006Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.003253147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5202a09b-51cd-458a-a4b5-1eba98659d6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.021279965Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.021326906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0bd55eeb-f8cb-480d-ada2-67cfa13a5bc1, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.041089791Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.041144941Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 079c91fb-c032-47d7-8d99-720e56571e78, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.06024651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.060315891Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd461cb3-4566-4efc-b780-d12158809749, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.078843244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.078882704Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 74918dc1-e131-42b3-a941-d060a1358fb7, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.096973233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.097028774Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7794a713-b4bd-4c00-82a2-314c5dcabb2a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8d354229-67d2-4778-8fa1-24cf4429afe5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.116132113Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.116175423Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4fcc4205-2962-474d-86f6-0c103405365d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.134867448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.134909389Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: efd17334-7981-45c7-aa72-f1707204494e, dataset_name: DatasetName { pool_name: ZpoolName { id: 857bd84b-5498-49a6-9512-c8f97ed577a9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.154032818Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.156445021Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a153857-b213-4886-b5ce-7d0c3c0f53de, dataset_name: DatasetName { pool_name: ZpoolName { id: 6016711c-532d-4238-a301-8a9cba14887f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.173340868Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.173387129Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 61767cb7-93e9-454e-9b1d-ef1bf115e261, dataset_name: DatasetName { pool_name: ZpoolName { id: f5178f34-f85d-4d2a-9f4f-5d19e04fae27 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.192249535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.192300926Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b3221346-1766-4a71-8cea-195049832d24, dataset_name: DatasetName { pool_name: ZpoolName { id: d0d8d3b6-4496-4772-9553-97dafea7d189 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.210816209Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.210864619Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 28ae2d32-273e-4c3c-9340-70dea8d888c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 16235c83-9817-4220-9cf4-0c733b95b0bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.229375262Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.229424913Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89153b11-b7aa-463f-8e21-1efa1222641d, dataset_name: DatasetName { pool_name: ZpoolName { id: 22f621aa-0a05-43b7-9d04-54301aba846e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.247726804Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.247777574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03a2b0b2-6036-4696-ac52-e5c06c2b5286, dataset_name: DatasetName { pool_name: ZpoolName { id: b038acd0-b651-49ff-832f-7b28f7bae91f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.266003045Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.266069525Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.266302808Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.266430869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.444182086Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.444374678Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.444390318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.444417589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.444429869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.444459819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.444476649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.444514079Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.467237464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.469354805Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.641478537Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.643119113Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.667993499Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.668250291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.670890448Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.670975668Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.674741415Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.674748346Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.674754826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.674956598Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.680741315Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.685939876Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.691221058Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.6963821Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.701169917Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.705886883Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.711243436Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.716470038Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.823622787Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.828207463Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.832909569Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.837510535Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.842242991Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.84717318Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.852106359Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.857381121Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.86227743Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.867652863Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.874528551Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.880753112Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.886834583Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.893008233Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.899811811Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.907318195Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.913980261Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.920219533Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.926410904Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:02.985612459Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.047177578Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.124812245Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.186288973Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.243893433Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.246457928Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.246479158Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.351669418Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 208, 116]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.351729399Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.3548946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.35493732Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.466642044Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 214, 233]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.466693205Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.469410562Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.469423942Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.590497859Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 207, 225]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.590554749Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.592929373Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.592942883Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.707884589Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 143, 16]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.7079382Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.711842219Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.711864369Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.805639623Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 136, 218]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.805690201Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.807331223Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.807355783Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.807643855Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.807656296Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.807763857Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.807774807Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.807923628Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.807934469Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.808013109Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.808030599Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.808184851Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.812409763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.812540344Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.81617068Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.819114319Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.81918425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.81919502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.822160969Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.825208729Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:03.83238614Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.345272791Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.354533602Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.362436291Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.369897965Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.461605441Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.476401847Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.742532498Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.755573927Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.770303323Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:04.783298912Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.552289769Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.552915506Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.886185561Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.886248581Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_b3221346-1766-4a71-8cea-195049832d24:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.903767795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.914094967Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.920234457Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.978520364Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:06.982466253Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.019894463Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.027176035Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.028416257Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.028479328Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.04287818Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.079600143Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.11264585Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.2441413Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.246712255Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 28.040628ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:07.302220164Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.326849778Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:07.328041609Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 132.039715ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:10.699157267Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 125.244045ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.151166436Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 48.248386ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.425156005Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 26.9206ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.467978847Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 33.50851ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.483555291Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:11.485710673Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 213.572545ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.503469418Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 37.113523ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.564206979Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 96.538966ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.694703269Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 52.96612ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.872300265Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 141.784417ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:11.94964924Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 63.280228ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.013641882Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 31.813607ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.041661579Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 55.48534ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.063621166Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:12.066691677Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:12.069464344Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:12.073480554Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.11030137s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.114250137Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 97.944538ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.148205623Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 293.75216ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.162842687Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 62.729016ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.195004615Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 68.301317ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.220079693Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 291.435985ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.253638185Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 156.936868ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.278336469Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 78.865696ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.299043014Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 126.617298ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.314164433Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:12.314297025Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 62.476215ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.337881554Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:12.337973177Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 136.326915ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.363985006Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 110.117852ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.379994564Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 155.313705ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.397088253Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 198.136904ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.411810089Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 96.615043ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.429862597Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 59.010086ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.447339849Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 489.827804ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.465941123Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 282.544819ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.484999452Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 462.305521ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.503582795Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 237.443756ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.523833705Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 144.722654ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.541003775Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 599.188251ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.581904959Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 339.353235ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.598239641Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 347.139799ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.613969667Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 592.312265ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.628064676Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 111.213698ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.644223676Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 44.565866ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.683969439Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 114.943255ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.699175189Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 40.166204ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.711232028Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 98.296199ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.755061391Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 111.964343ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.777044519Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 562.537753ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.795849755Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 344.273427ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.813785762Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 437.05471ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.829027373Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 323.477603ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.867965638Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 233.376699ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.879836805Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 126.42032ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.932303004Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 408.295527ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:12.976212088Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.033362088s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.011575608Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 733.67483ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.047845316Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 454.052694ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.066146607Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 322.655965ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.080814012Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 376.429545ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.130700105Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 536.850113ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.175552409Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 512.062406ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.203710497Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 573.981548ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.222512093Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 664.716628ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.239562092Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.402573139s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.256492369Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 1.131298035s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.280953341Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 426.216448ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.376410565Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.194738961s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.405988217Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 956.475527ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.428457879Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 438.521841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.524759531Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 789.191038ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.547080651Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:13.674160628Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:13.674714423Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:13.68957941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:13.696974133Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.227294051s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.728622346Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 506.17713ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.743443982Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.419128184s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.794911871Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 970.24874ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.809556546Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 1.404682044s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.825004989Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 729.265781ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.924467782Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 542.891889ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.980247744Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 54.0154ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:13.996696717Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 50.565744ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.009286211Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 996.290591ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.064226114Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 65.403167ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.270281041Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 93.124572ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.298251978Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 550.4311ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.316179425Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 208.728098ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.35514684Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 1.207655766s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.38850587Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.118690649s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.404808701Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 238.511631ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.426880359Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 659.180585ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.446754586Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 602.129418ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.537617454Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 217.340042ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.578847672Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 827.025163ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.61707415Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 1.210332807s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.669891972Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 212.761673ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.68378627Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 566.593794ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.698136701Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 649.536518ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.796379803Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.014462472s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.809968517Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.159322882s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.883423963Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 767.665451ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.913251648Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 755.292137ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:14.969717916Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.374544708s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.074331011Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 1.316096676s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.118601228Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 990.914346ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.154271071Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.198677691s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.228304343Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 938.735864ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.24214812Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.032135376s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.278965254Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.148932493s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.458123354Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 720.686081ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.469455087Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 977.902603ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.536065555Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 534.324258ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.608509471Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.481838008s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.694554272Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 554.003287ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.742349604Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.150804186s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.84403021Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:15.855057629Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:15.866040467Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:15.873174868Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:15.877253088Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:15.877309399Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:15.882567161Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 633.239668ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:15.898409107Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.404200149s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.057327478Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 62.166079ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.135541542Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 28.644339ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.144826514Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 44.439884ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.153335608Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 666.647641ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.162790371Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.429391559s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.172383866Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 74.29378ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.181986001Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 1.112867972s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.191180862Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 777.020252ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.198795827Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 87.582291ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.209653915Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 126.382964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.218010997Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 274.132478ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.267092812Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.466700506s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.307827805Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 193.541083ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.318023176Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 530.527003ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.32857005Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 137.452858ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.359573997Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 616.940499ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.440736359Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 1.181531162s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.453464074Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 749.871112ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.464599074Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 405.534302ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.510120734Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 203.796015ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.527483656Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.449467218s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.538574176Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 755.407189ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.550385272Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 387.405826ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.607878831Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 644.842265ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.618038991Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 1.136940454s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.799463675Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 529.167591ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.895899168Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.133817792s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.907877377Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 742.980053ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.927238668Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 1.314869237s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:16.988563465Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.094809729s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.005773355Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 580.660154ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.015611922Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 714.895375ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.068480715Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 780.754142ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.187920696Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.351505682s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.226003262Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 688.727458ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.278058957Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 512.88058ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.315902181Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 707.611288ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.326549726Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 538.02937ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.437335701Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 530.999132ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.474870152Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 870.693242ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.609781965Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 615.083961ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.621898985Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 736.153046ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.648134745Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 965.913153ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.683478834Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 585.257399ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.787053728Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.255615043s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.798872275Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 619.420369ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:17.866659405Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:17.92275299Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 1.226033172s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.006208005Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:18.006427757Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 673.989643ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.025596707Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.20630254s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.038540945Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.109212805s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.050409652Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 678.889245ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.062824225Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 1.224031861s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.074886794Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 1.190469848s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.086268856Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.046413378s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.097285065Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.275153209s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.115588486Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.461216001s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.25151866Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 26.802202ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.308038459Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 951.001847ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.320380031Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 126.797542ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.348346327Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 937.69149ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.385370224Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 661.181125ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.396525044Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 289.450806ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.48622703Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 649.124662ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.497561252Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 565.990317ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.508020665Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 977.208691ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.588493441Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 655.462885ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.66322644Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.110805896s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.714151753Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 871.370135ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.756076828Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 388.546424ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:18.785907963Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.232092849s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.068932361Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 897.706664ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.081145582Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 1.044801715s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.09013377Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.172522237s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.154458096Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 1.007675501s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.162428905Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.414456641s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.169589016Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 695.859646ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.181801267Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 639.627528ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.24080718Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.327547083s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.253671407Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 927.339456ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.278975707Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 850.902772ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.28629126Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 799.455949ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.307722362Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 516.00372ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.327176274Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.463465648s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.33483781Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.196234282s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.399657311Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 777.512666ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.575075304Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 559.134602ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.635777494Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.197404543s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.72135061Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 582.132291ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.834728571Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 517.625782ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.851472837Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 1.397274997s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.887604594Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 658.442853ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.895117888Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 710.210634ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:19.987940916Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 1.061756655s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.120514767Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 923.209554ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.153951487Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 923.435261ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.165161378Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 1.343984354s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.181567Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 542.808039ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.211783169Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 1.301985092s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.225568415Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.495070654s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.349122037Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 571.77559ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.365548109Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.123533041s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.381379286Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 1.188209236s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.391991491Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.407083328s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.429690323Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 606.041977ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.56498841Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 889.949407ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.593684454Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 930.393697ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.605114317Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 625.325164ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.619281337Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 1.379863029s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.712680431Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 500.870999ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.767655464Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.233450724s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.810285956Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.066351851s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.933447143Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 590.252281ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:20.950723444Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 617.18719ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:21.097757048Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 957.84113ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:21.106783547Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 727.952106ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:21.181163602Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 1.450474235s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:21.191109071Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 609.596148ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:21.255188194Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.446234827s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:21.276821938Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 980.871508ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:21.293529783Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 789.714082ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.378707981Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.397691851s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.387078564Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 525.945674ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.395112844Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 566.623501ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.402957701Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 903.142065ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.410750848Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 551.100005ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.418485215Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.131628409s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.426269672Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 732.932896ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.43425854Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 1.325431912s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.441839374Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 834.732348ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.449338859Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 621.360132ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.457095285Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 997.961212ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.465275236Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 883.89751ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.472444037Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 1.47727984s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.480629268Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.1055358s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.488157832Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 547.08921ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.495776898Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.491424538s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.503496614Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 839.47444ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.652549828Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 821.170741ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.73271328Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 699.426871ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.931143842Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 622.27211ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.978706933Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.433922243s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:22.986889693Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 785.914513ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.052381431Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.095651424s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.087081344Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 1.003644599s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.176387507Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.276202909s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.185438656Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 1.270720543s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.331751253Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 1.180681896s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.339875673Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 1.065125075s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.364949761Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.31973813s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.471903838Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 569.672532ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.500110087Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 727.595412ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.563689386Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.152910233s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.571844037Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 914.383768ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.603430539Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.196978048s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.610496811Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 1.149113455s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.638382535Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 567.747856ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.780346848Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.057915014s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.797625199Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 1.155865014s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.86145677Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 1.391021733s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:23.968172265Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.374732668s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.005668836Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 631.306827ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.079362514Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 929.925536ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.189524863Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.093933843s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.197462642Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.498611226s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.300648792Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 1.292527567s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.356214072Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.453400909s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.431082552Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 950.291294ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.438981249Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.122324242s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.470092347Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.457752375s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.526957189Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 1.059733075s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.53518585Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 766.022163ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.611978109Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 641.47902ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.719504972Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.440401123s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.732888385Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 585.577014ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.79614318Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.06200158s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.80321814Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 663.686231ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.817728934Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:24.835379948Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65342"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:24.8467025Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"c11a52d1-1ff0-4fa5-b106-9f14fcc2a13a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65342","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9884,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 651.957536ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:24.866460136Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 720.697942ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.028948622Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 1.295111527s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.132812609Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 824.456887ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.301124453Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 1.209513657s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.369335687Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 801.443021ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.378650279Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 680.681696ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.391717249Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 1.189691246s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.404824438Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 822.290802ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.413565414Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 706.986033ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.424517963Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 929.666601ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.49405454Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.13173434s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.531452379Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 870.26894ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.60335342Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 1.121767582s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.612164337Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 1.103888533s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.660933739Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 585.156805ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.761806167Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 888.458853ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.816621058Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 503.555513ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.878388989Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.414356602s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.895018054Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.455409913s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:25.999088073Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 1.022347624s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.083111383Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 919.046655ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.135892895Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 959.323599ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.150158406Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.350496754s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.184016271Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 1.125438568s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.192939529Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 870.7496ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.245262965Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 581.12015ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.357674948Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.110416424s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.392453691Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.127484385s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.43592393Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 828.116493ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.446801698Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 709.825512ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.484915625Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 957.385134ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.590885552Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 1.249775535s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.604678969Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 948.050709ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.677823022Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.157000762s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.719903908Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 1.185475618s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.74644133Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 708.018175ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.779281275Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 1.0227012s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:26.953902341Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.140158s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.068212262Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 845.339894ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.118810602Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 794.741794ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.131274975Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.305046703s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.142511056Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.082421964s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.211361767Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 1.303670864s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.296812882Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.436988447s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.323189862Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 722.486218ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.333953109Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 861.686674ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.471461357Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 738.668195ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.506183461Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 510.004713ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.526030827Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.491415839s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.551192016Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 893.424927ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.572413676Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 722.329548ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.591548285Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.256324803s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.660471586Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 615.611014ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.869859376Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.161409709s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.889981755Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 1.435184823s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.946532034Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 1.264243171s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.965133738Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 1.178509683s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.982005015Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 527.816769ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:27.998279696Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.252276731s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.054966036Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 721.962658ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.074038305Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.196714219s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.224647134Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 1.497696376s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.259268726Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 806.186634ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.309066049Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.123050203s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.326649432Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.05890358s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.344749911Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 701.523137ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.463102801Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 1.232668989s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.479841457Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 736.235307ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.498985566Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 793.598057ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.539211454Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 866.334023ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.610978233Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 626.331853ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.771322799Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 778.031711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.8088891Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.402175073s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:28.928955277Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 753.085105ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.052154255Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 705.807181ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.064548198Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 508.423795ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.125821543Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 837.051268ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.172599906Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.474141962s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.18313322Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 816.435332ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.239437817Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 585.085948ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.249316314Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 662.611774ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.317704321Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 633.506216ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.342600656Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 701.900651ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.392300498Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.248104847s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.408584409Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.122385592s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.419466587Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 739.000164ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.431561946Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 1.17262241s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.462186118Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 1.18897639s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.489961933Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 525.080262ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.59995679Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.455609987s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.647187957Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 1.206210117s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.72735965Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 1.35624207s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.769538357Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 631.71561ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.783595716Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 752.102406ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.820516151Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 834.803268ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.850145874Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 529.810361ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.988421521Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 787.258165ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:29.999367289Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 717.40093ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.022798951Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 873.975394ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.06719239Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 1.04746519s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.105146245Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 1.470494619s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.13803868Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.153701637s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.184706021Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 626.367414ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.342314539Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.471836363s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.43039561Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 1.273845505s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.538843552Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 1.179295942s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.558502846Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 544.568821ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.585880137Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 761.500615ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.652210442Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 1.127297091s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.668968538Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7\") failed: Property not found. retry in 1.185703947s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.681665054Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 807.160131ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.692854194Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 1.139906417s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.708222636Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 1.248323041s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.756630205Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 705.8179ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.802037004Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 1.040803897s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.946190219Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286\") failed: Property not found. retry in 1.267977617s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.957614022Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 1.373257133s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:30.983159784Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 595.147238ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.118916127Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 1.332389049s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.138896384Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 939.16072ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.151559599Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 782.420426ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.167180364Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.32005374s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.352563497Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 985.156983ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.427414789Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 854.982025ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.514964991Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.428851383s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.523998241Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d\") failed: Property not found. retry in 956.051914ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.621650106Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.425508236s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.725425332Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 579.953981ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.750086466Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 811.298201ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.807319132Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178\") failed: Property not found. retry in 1.441031542s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.823704044Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261\") failed: Property not found. retry in 1.479375823s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.858530078Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.133403878s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.916021804Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d\") failed: Property not found. retry in 1.474939014s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.958412906Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018\") failed: Property not found. retry in 1.345810378s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:31.997469992Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 610.213929ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.014888174Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 630.918717ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.106820213Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711\") failed: Property not found. retry in 867.990799ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.340139529Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 652.62287ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.366060206Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7\") failed: Property not found. retry in 973.374821ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.382007714Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24\") failed: Property not found. retry in 941.687075ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.426597204Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9\") failed: Property not found. retry in 781.138708ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.484105172Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749\") failed: Property not found. retry in 957.850248ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.628616051Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343\") failed: Property not found. retry in 588.019585ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.638186985Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1\") failed: Property not found. retry in 1.4314365s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.682368252Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a\") failed: Property not found. retry in 1.465518319s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.748373825Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78\") failed: Property not found. retry in 1.045217991s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:32.965548002Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de\") failed: Property not found. retry in 754.652652ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:33.033665905Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d\") failed: Property not found. retry in 1.388850523s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:33.062369399Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e\") failed: Property not found. retry in 1.233057652s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:33.162407798Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.49188801Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_efd17334-7981-45c7-aa72-f1707204494e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.491955761Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_5a153857-b213-4886-b5ce-7d0c3c0f53de"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.491962001Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_61767cb7-93e9-454e-9b1d-ef1bf115e261"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.491966971Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_4fcc4205-2962-474d-86f6-0c103405365d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.491999572Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_b3221346-1766-4a71-8cea-195049832d24"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492004281Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_89153b11-b7aa-463f-8e21-1efa1222641d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492008961Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_03a2b0b2-6036-4696-ac52-e5c06c2b5286"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492013102Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_bd461cb3-4566-4efc-b780-d12158809749"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492022922Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_7794a713-b4bd-4c00-82a2-314c5dcabb2a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492030832Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_74918dc1-e131-42b3-a941-d060a1358fb7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492043252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_f39546e4-8c7a-4ef8-9393-55f5c91af2c9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492047702Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_bf0746e7-31d1-46cc-aca3-2c04aaa44343"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492063982Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_28ae2d32-273e-4c3c-9340-70dea8d888c1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492070312Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_079c91fb-c032-47d7-8d99-720e56571e78"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492081762Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_ecaf7873-a899-4ed4-9a5d-5fa055a6991d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492087362Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_f4013866-e0b3-4697-9c6d-9fea86946018"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492095173Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_36b5db01-3ec6-4b79-8cd0-3bc225315711"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492099253Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_bf77d87d-9513-4b4d-9600-4f57755c3178"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.492104022Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_9b3d43d8-586a-4880-98e4-d4c4925a2ac7"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.493155123Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.515343192Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.534153808Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"47ce7e9a-c9d4-4dbb-8eb2-6497d3f9c841","remote_addr":"[fd00:1122:3344:101::1]:53153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":32647813,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.618615403Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1235"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.621917496Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.622247949Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.63247517Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:679"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.664497687Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:692"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:34.721401989Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:816"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:36.150140324Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:36.173286703Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:36.174159121Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:36.236141084Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:36.367485753Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:36.559981565Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:36.808389991Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:36.831119716Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:37.041433665Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:37.172799994Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:37.184040825Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:37.370403927Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"95.125705719s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:37.419668344Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:37.879348018Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:38.187956729Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:38.228232197Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:38.527598057Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:38.824045138Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:39.195925955Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:39.466311797Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:40.199128382Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:40.520541875Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:40.825373683Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:41.202868395Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:41.676581638Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:42.20581275Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:42.762790766Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:42.824605977Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:43.20927149Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:44.21274971Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/65790f8d-3cb9-4324-9682-a18211fa9b99/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:44.629400369Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:52:44.823624069Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/4115e24c-e8db-4625-994c-3c144db05bf3): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:52:45.215952928Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ea220ea-649b-4b59-95fa-2f790727f3e5\", \"content-length\": \"173\", \"date\": \"Sun, 05 May 2024 01:52:45 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"65790f8d-3cb9-4324-9682-a18211fa9b99\\\"\", request_id: \"3ea220ea-649b-4b59-95fa-2f790727f3e5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:45.61984781Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6f0ed97d-683f-45ae-8149-89b57571533b\", \"content-length\": \"173\", \"date\": \"Sun, 05 May 2024 01:52:46 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"4115e24c-e8db-4625-994c-3c144db05bf3\\\"\", request_id: \"6f0ed97d-683f-45ae-8149-89b57571533b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:47.050710895Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:878"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:48.889051949Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:49.020781091Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:61075"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:49.021873772Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:49.108839362Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"f3676ea8-31e3-465b-bc23-3d0b9e3be34d","remote_addr":"[fd00:1122:3344:101::a]:61075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87918,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:49.109939372Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:49.110119294Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:49.112090014Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:49.11274694Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"fc9ee813-aba9-4c4d-a549-a1ce6249a854","remote_addr":"[fd00:1122:3344:101::a]:61075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3622,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.023663925Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:45114"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.028906847Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.029363012Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.029403532Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.029432992Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.029439732Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.029445172Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.029462222Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.029497953Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d2878890-38e4-4ffb-8c7d-d5fbe69d4416","remote_addr":"[fd00:1122:3344:101::a]:45114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4447,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.031592254Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:888"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.279161641Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58881"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.279390604Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.368403604Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"07d72384-1f9d-4e46-bc9c-7d66a368aebb","remote_addr":"[fd00:1122:3344:101::a]:58881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89113,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.368707616Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.368775157Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.3701069Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:50.370462074Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"065d9735-04f9-489e-9249-0363b88f9031","remote_addr":"[fd00:1122:3344:101::a]:58881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1834,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695139929Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39953"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695401452Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695455203Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695496293Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695512493Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695526823Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695532643Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695553284Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.695575244Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0113ac09-adc1-4ea9-b203-a9bc1dc8bb4b","remote_addr":"[fd00:1122:3344:101::a]:39953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":378,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736085974Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:43445"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736319477Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736368327Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736407028Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736435318Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736465438Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736474938Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736480378Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.736496598Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"5db7693f-d088-43fe-a7a0-5c8b24a03ca5","remote_addr":"[fd00:1122:3344:101::c]:43445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":308,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.834924902Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38082"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.835182114Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.835216014Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.835250345Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.835262455Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.835275795Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.835282745Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.835968392Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:51.835984542Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"00f46a34-4ab9-4dc8-9eb7-19948386a439","remote_addr":"[fd00:1122:3344:101::b]:38082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":918,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:54.886015135Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:52:54.886147016Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"572cd2c2-3931-4dfa-8bbb-a97d0395f695","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61444","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.774200195Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59791"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.850701551Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"1eec58da-30aa-4090-96dc-724d6cd64e9e","remote_addr":"[fd00:1122:3344:101::c]:59791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":76419,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.851328037Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"17035d36-005e-46f3-b59a-3aec52fea9f2","remote_addr":"[fd00:1122:3344:101::c]:59791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":361,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.861485648Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47247"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.913406811Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"598f91c5-a29b-4495-a121-f5573f164fe3","remote_addr":"[fd00:1122:3344:101::a]:47247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51814,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.913922656Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9dddef3e-651d-4314-a2e3-c979aaac19af","remote_addr":"[fd00:1122:3344:101::a]:47247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":234,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.914438481Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.914464832Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.91546445Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.919651883Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:15.919847705Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"2dd5d6a2-b6b7-4844-a68c-bb86db8e9e0b","remote_addr":"[fd00:1122:3344:101::c]:59791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:16.039363047Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:44549"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:16.08422011Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b808253d-e291-4ab5-97c4-d73f7aa6c7d5","remote_addr":"[fd00:1122:3344:101::b]:44549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44814,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:16.084499873Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5b24f617-b9f2-49e8-b618-3b74aca043c2","remote_addr":"[fd00:1122:3344:101::b]:44549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:24.932452083Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:24.932569654Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"965122b2-5a9d-4252-b382-cd7ebd78c5ec","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64738","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:45.911027848Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:37153"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:45.958151674Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e1722f7c-7ecb-46ea-a498-56345652b640","remote_addr":"[fd00:1122:3344:101::a]:37153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47072,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:45.959311425Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b5713272-fd92-4b96-883b-c4912aa30aeb","remote_addr":"[fd00:1122:3344:101::a]:37153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1013,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:46.003431182Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34407"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:46.05077546Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"813356fb-7ecf-4f2b-a342-fe09b1907cbc","remote_addr":"[fd00:1122:3344:101::c]:34407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47236,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:46.051006572Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a2e52206-e8f9-47ad-8aef-1b7a1c225f6b","remote_addr":"[fd00:1122:3344:101::c]:34407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:46.071138621Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:46.116808543Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f4e26e08-9bc3-4b29-ac71-86c97a64870f","remote_addr":"[fd00:1122:3344:101::b]:50971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45574,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:46.117136076Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e0c6b18d-df8e-429d-a43e-1f20056767d7","remote_addr":"[fd00:1122:3344:101::b]:50971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:54.965362351Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:53:54.965459302Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"cbc13e3f-a866-475d-aa13-5ceed1afafb8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45467","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.662528676Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57544"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.70744589Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d29728c5-c665-4139-9974-439da30e723e","remote_addr":"[fd00:1122:3344:101::a]:57544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44865,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.707627452Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:62550"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.707714663Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57190"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.710777263Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"fcf5047d-5126-4e07-9bed-290acb64b4c8","remote_addr":"[fd00:1122:3344:101::a]:57544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2885,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.745332325Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"504881ea-8011-4ecf-b7f8-d5f1eaa695a4","remote_addr":"[fd00:1122:3344:101::b]:62550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":37578,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.748162523Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5a13c065-47a7-4313-bde9-6c268710e014","remote_addr":"[fd00:1122:3344:101::b]:62550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":379,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.787230639Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ac86256e-c08f-4db8-9ec8-5952253cc209","remote_addr":"[fd00:1122:3344:101::c]:57190","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":79363,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:15.787544502Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1445f702-f660-4588-9435-5b3c7909f6ea","remote_addr":"[fd00:1122:3344:101::c]:57190","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:24.998599894Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:24.999729166Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ca035748-5ea6-402b-8d37-475fa34eec91","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34020","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1091,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:38.44923782Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:42.610595889Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:49292"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:42.618432937Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"81ffae44-0088-417a-84c5-4d903d30edc8","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6979,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.812800957Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33286"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.861059934Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"03b4c67f-b07a-4f3a-89bf-9c3d96f9980d","remote_addr":"[fd00:1122:3344:101::a]:33286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48211,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.861280266Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0d4b0d12-c575-4054-a22f-67770cd8ddc6","remote_addr":"[fd00:1122:3344:101::a]:33286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.871243365Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63043"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.91826133Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"cb6b58ae-1343-4b24-9a03-8a950ed8f3f8","remote_addr":"[fd00:1122:3344:101::c]:63043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46931,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.918524722Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c0435e14-15de-4c46-af83-b2492bef301b","remote_addr":"[fd00:1122:3344:101::c]:63043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.94258265Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:51235"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.984125726Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"7c5a7b52-293b-4a51-aedf-059343350123","remote_addr":"[fd00:1122:3344:101::b]:51235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41502,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:45.984348731Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"aa81ba22-005f-43c0-948f-d2925f75e680","remote_addr":"[fd00:1122:3344:101::b]:51235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:55.032529956Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41136"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:54:55.032602507Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f239737d-9b05-4a23-9e54-baa1d521cfa1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41136","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":64,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:12.61026833Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"839a70eb-208b-48e7-8184-d5a3b906f962","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":509,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.636345486Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:45541"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.684052668Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"24396933-907b-48a7-8b47-fea95f416513","remote_addr":"[fd00:1122:3344:101::a]:45541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47647,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.684178459Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:43570"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.687621953Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d2401717-02ee-4df2-b77d-de326f6f7299","remote_addr":"[fd00:1122:3344:101::a]:45541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3279,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.705990355Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.732222614Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"bd4934a6-d2a5-4807-a5e8-d4ee34654932","remote_addr":"[fd00:1122:3344:101::c]:43570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47927,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.735768779Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3db96854-69bc-4980-baeb-cb037f91efe3","remote_addr":"[fd00:1122:3344:101::c]:43570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":611,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.776903176Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d30a4b1c-6101-40ae-b68a-2928dda7c75b","remote_addr":"[fd00:1122:3344:101::b]:33231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":70817,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:15.777240639Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b591433f-1e9d-4c61-89d0-4ec67c01f4a4","remote_addr":"[fd00:1122:3344:101::b]:33231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:25.065715968Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41269"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:25.065813279Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"718e6198-9d21-4ebc-a076-be5210ec4e19","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41269","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":82,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:42.609812502Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"daca9db6-7dfc-4ad6-917b-c48ae0fe13be","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":385,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:45.957791021Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46559"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:46.002934917Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b23a795b-9507-45da-8195-9e27b5cf79c1","remote_addr":"[fd00:1122:3344:101::a]:46559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45058,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:46.00420756Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a15919df-cba7-4c2f-a11f-cef21336d170","remote_addr":"[fd00:1122:3344:101::a]:46559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1026,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:46.105714373Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33445"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:46.152802549Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0c413741-27ca-4a4b-b9b1-841d24117040","remote_addr":"[fd00:1122:3344:101::b]:33445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47055,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:46.15291368Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:45007"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:46.156272153Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e5c88ec5-1512-4199-a343-6bc6a2a4fd7f","remote_addr":"[fd00:1122:3344:101::b]:33445","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3140,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:46.194477151Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e4eaca00-0b02-40db-afe4-968ba1e7b724","remote_addr":"[fd00:1122:3344:101::c]:45007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41464,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:46.194799754Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"2a7c9f9a-dbe9-4f72-80c3-332f07b1e85a","remote_addr":"[fd00:1122:3344:101::c]:45007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:55.098917263Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46996"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:55:55.09960365Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"5b3f9c8d-c97c-44fb-892f-59c3b92b5e5a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46996","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":648,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:12.609994652Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"1e571a86-8803-425f-bc49-4ba38fbac5a2","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":419,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.72222416Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:42499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.768592228Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"69e71b1f-94b6-454a-b8b3-07fbee5dee7f","remote_addr":"[fd00:1122:3344:101::c]:42499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46310,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.76876782Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:50989"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.768886161Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54387"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.769151924Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6813d1fb-458c-4aa9-8dc7-37946c936b31","remote_addr":"[fd00:1122:3344:101::c]:42499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":281,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.813599393Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ddc9c72c-4622-49ec-9c56-7e8635fa3b93","remote_addr":"[fd00:1122:3344:101::a]:50989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44651,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.816489272Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"867d30d1-4ed4-498d-838a-52d9c0a9935c","remote_addr":"[fd00:1122:3344:101::a]:50989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":582,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.856924352Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a48bbaf8-f5e8-4929-826e-e2213e1a7823","remote_addr":"[fd00:1122:3344:101::b]:54387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87828,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:15.857265425Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b56419c0-11af-4264-b642-0a94d166f3a2","remote_addr":"[fd00:1122:3344:101::b]:54387","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:25.131727674Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39195"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:25.132543352Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"7cf02782-52c4-4aba-8d8f-b4e7d1023614","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39195","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":772,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:42.619257397Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"afbd7bfb-3bf2-43e2-b058-92218b7eb459","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":387,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:45.989844309Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:46.040040546Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"fbf08719-4cdf-43e4-a808-e0f5c1d00059","remote_addr":"[fd00:1122:3344:101::c]:56400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50132,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:46.040228437Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:46.04051854Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"90cab195-a5d0-4ed1-9834-43b03df8b116","remote_addr":"[fd00:1122:3344:101::c]:56400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":249,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:46.061718565Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:46.08701405Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"249bb9db-0a25-410b-975b-3a9e367847de","remote_addr":"[fd00:1122:3344:101::b]:34233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46652,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:46.090429504Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7e2be952-c880-491f-ac46-36e71a06f286","remote_addr":"[fd00:1122:3344:101::b]:34233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":410,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:46.130584501Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5a515416-c3f4-41d4-a8de-a2fec735ca3c","remote_addr":"[fd00:1122:3344:101::a]:58014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":68749,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:46.130948964Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"20f8cc54-fa3b-495f-b971-08b804e4f7d0","remote_addr":"[fd00:1122:3344:101::a]:58014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:55.163751855Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45398"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:56:55.163864896Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"e9e4157d-d93e-4468-a097-b2f51bdc5e84","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45398","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:12.670266756Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"5443fff4-7ff3-4b6e-b2e0-cc654b65ce9c","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2399,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.731186266Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55365"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.777237981Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"695cad6a-47a1-4501-b36a-946721c1cca5","remote_addr":"[fd00:1122:3344:101::a]:55365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45988,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.777405373Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53932"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.777499524Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51330"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.781459143Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5f7dc96f-cbc6-43ca-909b-4f807aec8600","remote_addr":"[fd00:1122:3344:101::a]:55365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3830,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.822343437Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6cfb6255-d100-4eab-888e-6a1b0770b825","remote_addr":"[fd00:1122:3344:101::b]:53932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44847,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.826088514Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"11c5540a-c217-47cb-b296-80cd1065f2b9","remote_addr":"[fd00:1122:3344:101::b]:53932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":224,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.867810917Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"36147bf0-65b8-43b0-b1b0-e8f36132b163","remote_addr":"[fd00:1122:3344:101::c]:51330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90106,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:15.868799567Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9728ee63-8ed2-4a18-8358-2920d971d68a","remote_addr":"[fd00:1122:3344:101::c]:51330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":731,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:25.197620503Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43685"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:25.198258559Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"bc0a5f2f-ec85-4919-b41e-0ac0da6aafdc","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43685","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":626,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:42.668547073Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"4384b14a-ed02-4f8c-9310-ce515d915375","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":375,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.73818464Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:61224"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.738433033Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.738479683Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.738519714Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.738539824Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.738547014Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.738558624Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.738574524Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.738592055Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"32291d39-2d18-429c-9838-2a0dc091f0d4","remote_addr":"[fd00:1122:3344:101::c]:61224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":380,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.797519457Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58253"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.798517167Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.798548157Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.798574998Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.798591218Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.798601148Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.798618228Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.798635788Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.798649818Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ba84c4e5-2300-44b0-bd8b-8e31a55fb33b","remote_addr":"[fd00:1122:3344:101::a]:58253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1084,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.865628651Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:33621"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.915777296Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"82b94c72-9ac5-4018-b3c6-391f42b1d8b2","remote_addr":"[fd00:1122:3344:101::c]:33621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50097,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.915917218Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:62435"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916297341Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 01c551f1-31f5-43c7-b223-32301631ccff, kind: Service { id: bd461cb3-4566-4efc-b780-d12158809749 }, name: Name(\"external-dns-bd461cb3-4566-4efc-b780-d12158809749\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 208, 116])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 35d1645c-f03c-4d19-994d-7278c593c23c, kind: Service { id: 079c91fb-c032-47d7-8d99-720e56571e78 }, name: Name(\"external-dns-079c91fb-c032-47d7-8d99-720e56571e78\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 16])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0bc7bb77-b8a1-40b7-b397-32dc9ae91527, kind: Service { id: bf0746e7-31d1-46cc-aca3-2c04aaa44343 }, name: Name(\"nexus-bf0746e7-31d1-46cc-aca3-2c04aaa44343\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 225])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 2f87452f-8947-4eeb-bd4b-aa3e4a2b14ac, kind: Service { id: f4013866-e0b3-4697-9c6d-9fea86946018 }, name: Name(\"nexus-f4013866-e0b3-4697-9c6d-9fea86946018\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 136, 218])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: deb2d63f-99e6-4458-8546-6208b9b3f298, kind: Service { id: f39546e4-8c7a-4ef8-9393-55f5c91af2c9 }, name: Name(\"nexus-f39546e4-8c7a-4ef8-9393-55f5c91af2c9\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 233])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916357862Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f31da7e7-faad-4ebc-889e-eee5f52acc8c","remote_addr":"[fd00:1122:3344:101::c]:33621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":247,"response_code":"200"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916371312Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916418723Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916446613Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916466263Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916486663Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916505054Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.916519944Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"11507256-d96c-4329-9b3d-d3e588c6a7a7","remote_addr":"[fd00:1122:3344:101::b]:62435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":510,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.929340981Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38395"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.973349076Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"65f2b815-7225-4b63-8414-c6002d17f900","remote_addr":"[fd00:1122:3344:101::b]:38395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43943,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:45.974248194Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"97d7e468-e25b-44c1-8b14-f9fa3c88c65d","remote_addr":"[fd00:1122:3344:101::b]:38395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":760,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:46.063545297Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56507"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:46.107685034Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"8ef03147-091f-4545-85d3-7d9365362a80","remote_addr":"[fd00:1122:3344:101::a]:56507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44051,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:46.108396241Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a6b7db37-d34c-430a-bcc7-ca0a020369d0","remote_addr":"[fd00:1122:3344:101::a]:56507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":575,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:55.231589665Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:57:55.231680026Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"be100aab-8d32-4054-8906-e0ea72a0cf56","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61487","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:12.669267004Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"3f3862a2-bbaa-4cf8-9838-e49a989e7871","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":979,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.64274411Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.693123397Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a76fc570-09fe-4425-94fd-42456c972e4f","remote_addr":"[fd00:1122:3344:101::b]:48735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50319,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.693228949Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51575"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.736750019Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d664281e-0293-4327-8eba-6ddbb3113838","remote_addr":"[fd00:1122:3344:101::c]:51575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43463,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.737724578Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f5cac05f-5268-4ad3-93af-2701f3dcfad0","remote_addr":"[fd00:1122:3344:101::b]:48735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":871,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.737936271Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e79e8ca1-4e40-4067-a3fc-0f6c9f2ec863","remote_addr":"[fd00:1122:3344:101::c]:51575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.786109307Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.829235243Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"3a287052-1972-436b-93c5-58099e4c18a2","remote_addr":"[fd00:1122:3344:101::a]:60396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43078,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:15.829470465Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b551efaa-09d2-4e02-b624-5eede6bf9dfb","remote_addr":"[fd00:1122:3344:101::a]:60396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:25.2643281Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51626"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:25.264446631Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b462f044-173c-4c12-be95-2253cfc8bc0f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51626","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":83,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:28.219677646Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:28.219756967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:28.257710232Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999995011s","next_cleanup":"Instant { tv_sec: 3594, tv_nsec: 734869365 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:42.670759932Z","hostname":"sock","pid":1590,"uri":"//4115e24c-e8db-4625-994c-3c144db05bf3","method":"GET","req_id":"a9123aaa-1ded-41c2-8f9d-1afb4538da95","remote_addr":"[fd00:1122:3344:101::d]:49292","local_addr":"[fd00:1122:3344:101::1]:34981","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"4115e24c-e8db-4625-994c-3c144db05bf3","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1355,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.645134554Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60123"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.653675169Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: b0c7ba0e-fcb5-4129-a43c-4c342f11ac82, project_id: ea532787-eb2b-47bd-90aa-06c655ebc8b0 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-05T01:58:43.128417Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(e054e2f7-6ac1-4691-b801-cc32fd24570d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-05T01:58:43.179159708Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 47f3d011-75bc-4858-9b70-761c454f9437, kind: Instance { id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 163, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7230192), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 47f3d011-75bc-4858-9b70-761c454f9437, kind: Instance { id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 163, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7230192), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 47f3d011-75bc-4858-9b70-761c454f9437, kind: Instance { id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 163, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7230192), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(7230192))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 47f3d011-75bc-4858-9b70-761c454f9437, kind: Instance { id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 163, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7230192), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-26b707e9ec0\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: ee041cec-127a-4da1-9936-6ab1a645d7bb, read_only_parent: Some(Volume { block_size: 512, id: 3a25c4b4-4947-4f4c-b31c-d1ca74b13499, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0c005f84-f19c-4882-b4b7-9aeb435528cd, key: Some(\"jkqPi9p1Dj0Eoj/W03ciAIJc2UqOjzZQW0iaOLpsy9Q=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ee041cec-127a-4da1-9936-6ab1a645d7bb, key: Some(\"lNXX1Um9aaz5XZxxfNQUgsPgn7DdhiyhvgVg6wa5HKw=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACgVQ+lWKVYAABVD6VYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAoFUPpVilWAAAVQ+lWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxNWM1ZDE5OS1lZjc3LTQ5N2UtYmFjMS1mMWI5YmE2ZmUyMWMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJS3gxS01YdU85NGRKOCsvZUVEeHVldXhDNkhKdWxqOVlJcmJtdXV4T1RMSSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.65377855Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.654145114Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 47f3d011-75bc-4858-9b70-761c454f9437, kind: Instance { id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 163, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7230192), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 47f3d011-75bc-4858-9b70-761c454f9437, kind: Instance { id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 163, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7230192), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 47f3d011-75bc-4858-9b70-761c454f9437, kind: Instance { id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 163, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7230192), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(7230192))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 47f3d011-75bc-4858-9b70-761c454f9437, kind: Instance { id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 249, 163, 126])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7230192), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-26b707e9ec0\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: ee041cec-127a-4da1-9936-6ab1a645d7bb, read_only_parent: Some(Volume { block_size: 512, id: 3a25c4b4-4947-4f4c-b31c-d1ca74b13499, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0c005f84-f19c-4882-b4b7-9aeb435528cd, key: Some(\"jkqPi9p1Dj0Eoj/W03ciAIJc2UqOjzZQW0iaOLpsy9Q=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ee041cec-127a-4da1-9936-6ab1a645d7bb, key: Some(\"lNXX1Um9aaz5XZxxfNQUgsPgn7DdhiyhvgVg6wa5HKw=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACgVQ+lWKVYAABVD6VYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAoFUPpVilWAAAVQ+lWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxNWM1ZDE5OS1lZjc3LTQ5N2UtYmFjMS1mMWI5YmE2ZmUyMWMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJS3gxS01YdU85NGRKOCsvZUVEeHVldXhDNkhKdWxqOVlJcmJtdXV4T1RMSSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(e054e2f7-6ac1-4691-b801-cc32fd24570d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-05T01:58:43.179159708Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-05T01:58:43.128417Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.655105843Z","hostname":"sock","pid":1590,"uri":"/instances/15c5d199-ef77-497e-bac1-f1b9ba6fe21c","method":"PUT","req_id":"82a32e72-60bb-4bfa-ae09-d9c0d10d2d9a","remote_addr":"[fd00:1122:3344:101::a]:60123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9926,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.747698268Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:48552"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.748960931Z","hostname":"sock","pid":1590,"uri":"/instances/15c5d199-ef77-497e-bac1-f1b9ba6fe21c/state","method":"PUT","req_id":"aac6bcda-d228-46da-847a-e6da118325fc","remote_addr":"[fd00:1122:3344:101::a]:48552","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1176,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.769712006Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 249, 163, 126]), slot: 0, vni: Vni { inner: 7230192 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.770367283Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.802210887Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:43.823555669Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:45.090613855Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:45.090835697Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 74.559947ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:45.280113218Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 101.028616ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:45.372072358Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 247.674102ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:45.519735477Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 335.441664ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:45.776819788Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:45.8822209Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:57334"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:45.934370506Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"503f11b6-68ad-44d8-9e51-865f1aa5e619","remote_addr":"[fd00:1122:3344:101::c]:57334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52041,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:45.934691869Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3a475c63-5ab1-43ae-a62b-b0744a0f6bb4","remote_addr":"[fd00:1122:3344:101::c]:57334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:45.956847748Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:46.000898434Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0851defb-1dd8-45ff-b2e4-f77278a021ff","remote_addr":"[fd00:1122:3344:101::a]:58751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43973,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:46.001131546Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6885ac6f-2ba2-4b48-827b-f05a5051552b","remote_addr":"[fd00:1122:3344:101::a]:58751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:46.00859613Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34351"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:46.054063689Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f75685e8-714f-409a-965b-e0a08b6f6d9f","remote_addr":"[fd00:1122:3344:101::b]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45368,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:46.054334472Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"613dd87c-df6b-4992-8e37-2b461610f7c1","remote_addr":"[fd00:1122:3344:101::b]:34351","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 976.785837ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:46.122106242Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 1.229391093s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:47.109373202Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 647.479215ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:48.349881417Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 1.262676293s","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:49.006928442Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.297977685Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 36.677695ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:50.306176977Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\") failed: Property not found. retry in 130.446025ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:58:50.353594895Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.493550209Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.526160341Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACgVQ+lWKVYAABVD6VYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAoFUPpVilWAAAVQ+lWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxNWM1ZDE5OS1lZjc3LTQ5N2UtYmFjMS1mMWI5YmE2ZmUyMWMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJS3gxS01YdU85NGRKOCsvZUVEeHVldXhDNkhKdWxqOVlJcmJtdXV4T1RMSSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-26b707e9ec0\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: ee041cec-127a-4da1-9936-6ab1a645d7bb, read_only_parent: Some(Volume { block_size: 512, id: 3a25c4b4-4947-4f4c-b31c-d1ca74b13499, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0c005f84-f19c-4882-b4b7-9aeb435528cd, key: Some(\"jkqPi9p1Dj0Eoj/W03ciAIJc2UqOjzZQW0iaOLpsy9Q=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ee041cec-127a-4da1-9936-6ab1a645d7bb, key: Some(\"lNXX1Um9aaz5XZxxfNQUgsPgn7DdhiyhvgVg6wa5HKw=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::14]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 15c5d199-ef77-497e-bac1-f1b9ba6fe21c, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: ea532787-eb2b-47bd-90aa-06c655ebc8b0, silo_id: b0c7ba0e-fcb5-4129-a43c-4c342f11ac82 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.527594496Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.637631063Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-05T01:58:50.638151239Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.638646724Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.638659974Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-05T01:58:50.638151239Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(e054e2f7-6ac1-4691-b801-cc32fd24570d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-05T01:58:43.179159708Z }","propolis_id":"e054e2f7-6ac1-4691-b801-cc32fd24570d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.638672014Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(e054e2f7-6ac1-4691-b801-cc32fd24570d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-05T01:58:43.179159708Z }, propolis_id: e054e2f7-6ac1-4691-b801-cc32fd24570d, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-05T01:58:50.638151239Z } }","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-05T01:58:50.689699958Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.689718269Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.689735639Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-05T01:58:50.689699958Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(e054e2f7-6ac1-4691-b801-cc32fd24570d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-05T01:58:43.179159708Z }","propolis_id":"e054e2f7-6ac1-4691-b801-cc32fd24570d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:50.689744299Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(e054e2f7-6ac1-4691-b801-cc32fd24570d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-05T01:58:43.179159708Z }, propolis_id: e054e2f7-6ac1-4691-b801-cc32fd24570d, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-05T01:58:50.689699958Z } }","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:55.306811103Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56058"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:58:55.306947485Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"7bd50000-ae69-4dde-9796-6a6dd25d0df9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56058","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.214117864Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.215290216Z","hostname":"sock","pid":1590,"uri":"/instances/15c5d199-ef77-497e-bac1-f1b9ba6fe21c/state","method":"PUT","req_id":"d060ff87-6ae0-4a5f-92b4-432083277657","remote_addr":"[fd00:1122:3344:101::a]:60340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1120,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-05T01:59:01.215398197Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.215421917Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.215443747Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-05T01:59:01.215398197Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(e054e2f7-6ac1-4691-b801-cc32fd24570d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-05T01:58:43.179159708Z }","propolis_id":"e054e2f7-6ac1-4691-b801-cc32fd24570d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.215466757Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(e054e2f7-6ac1-4691-b801-cc32fd24570d), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-05T01:58:43.179159708Z }, propolis_id: e054e2f7-6ac1-4691-b801-cc32fd24570d, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-05T01:59:01.215398197Z } }","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-05T01:59:01.237331694Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.237348104Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.237364544Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-05T01:59:01.237331694Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-05T01:59:01.237331694Z }","propolis_id":"e054e2f7-6ac1-4691-b801-cc32fd24570d"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.237388394Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.237394334Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.237596396Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.237632697Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/76f4a639-a5a9-4d2e-b5ec-8f4d6b2d2f34/debug/bundle/zone\", \"/pool/int/fd584376-2139-48aa-bdc0-23bc960f5024/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/8d354229-67d2-4778-8fa1-24cf4429afe5/crypt/debug\", \"/pool/ext/0dd09cf1-d363-426a-8eda-5e5b5d2f8e2f/crypt/debug\", \"/pool/ext/857bd84b-5498-49a6-9512-c8f97ed577a9/crypt/debug\", \"/pool/ext/6016711c-532d-4238-a301-8a9cba14887f/crypt/debug\", \"/pool/ext/f5178f34-f85d-4d2a-9f4f-5d19e04fae27/crypt/debug\", \"/pool/ext/d0d8d3b6-4496-4772-9553-97dafea7d189/crypt/debug\", \"/pool/ext/16235c83-9817-4220-9cf4-0c733b95b0bd/crypt/debug\", \"/pool/ext/22f621aa-0a05-43b7-9d04-54301aba846e/crypt/debug\", \"/pool/ext/b038acd0-b651-49ff-832f-7b28f7bae91f/crypt/debug\"] }","zone_name":"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:01.878028787Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d\", bundle_id: aef2630e-e9e4-4c7d-bbe6-5c84e8536b21 }, time_created: 2024-05-05T01:59:01.237927839Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_e054e2f7-6ac1-4691-b801-cc32fd24570d","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:59:01.878086288Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:03.636608263Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:03.661492999Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-05T01:59:01.237331694Z }, propolis_id: e054e2f7-6ac1-4691-b801-cc32fd24570d, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-05T01:59:03.661438659Z } }","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:03.73537609Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:49622"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-05-05T01:59:03.735551642Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:03.735567402Z","hostname":"sock","pid":1590,"uri":"/v2p/47f3d011-75bc-4858-9b70-761c454f9437","method":"DELETE","req_id":"c3a6fbb8-8382-4a04-bc5a-1b3dec07160b","remote_addr":"[fd00:1122:3344:101::a]:49622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":124,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:04.092292489Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-05T01:59:01.237331694Z }, propolis_id: e054e2f7-6ac1-4691-b801-cc32fd24570d, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-05T01:59:03.661438659Z } }","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-05T01:59:04.095153887Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"} {"msg":"Failed to publish state to Nexus, will not retry: Notification(Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"507359b7-716f-416f-b024-3e4e4854b70c\", \"content-length\": \"177\", \"date\": \"Sun, 05 May 2024 01:59:04 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: instance with id \\\"15c5d199-ef77-497e-bac1-f1b9ba6fe21c\\\"\", request_id: \"507359b7-716f-416f-b024-3e4e4854b70c\" })","v":0,"name":"SledAgent","level":50,"time":"2024-05-05T01:59:04.136768028Z","hostname":"sock","pid":1590,"instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c","component":"InstanceManager","file":"sled-agent/src/instance.rs:555","instance_id":"15c5d199-ef77-497e-bac1-f1b9ba6fe21c"}