[ May 14 09:39:02 Enabled. ]
[ May 14 09:39:02 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ May 14 09:39:02 Method "start" exited with status 0. ]
[ May 14 09:39:02 Rereading configuration. ]
[ May 14 09:39:02 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:02.796641187Z","hostname":"sock","pid":1590,"file":"illumos-utils/src/running_zone.rs:163"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:02.796821061Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:02.853098466Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:02.853135247Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:02.969729898Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:02.978678068Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:02.986621021Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.588519067Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.725364999Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:119"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.72541031Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:131"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.725465041Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:141"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.725588205Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.725612665Z","hostname":"sock","pid":1590,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.725627956Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.725646396Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.728451828Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:03.728464678Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.728487819Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:176"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.728492409Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.72851598Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.728554321Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.728607592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.728624513Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.728653233Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.741181885Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.741213276Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"112d7827-193f-456c-94a5-19163d29475e (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:03.842487044Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.044317533Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.044354124Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.044360024Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.044370014Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.044375784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.044381304Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.044400185Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.053233562Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.053253772Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.14241759Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356194676Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356228857Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356248727Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356259218Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356265508Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356276578Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356316609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356323269Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.35634686Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356367901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356372041Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356395031Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356429772Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356434202Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356449103Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356452313Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356456553Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356466003Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356470043Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356474933Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356488394Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356497074Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356510314Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356525035Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356529145Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356532605Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356536585Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356545495Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356562315Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356566216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356575616Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356581026Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356597816Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356610077Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356613337Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356616947Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356641488Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:91"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356655078Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:93"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356662088Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356681369Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356692299Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:200"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356751971Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356765581Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356776301Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356785781Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356805732Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356823012Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356827712Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:205"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.356835393Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:220"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.367737512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.367775233Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999944149s","next_cleanup":"Instant { tv_sec: 1353, tv_nsec: 439613001 }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.367813794Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.367821704Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.367862316Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.367878466Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371193031Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371292793Z","hostname":"sock","pid":1590,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.2.164,\n                last: 10.151.2.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.2.164,\n        10.151.2.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV1 {\n        rack_subnet: Ipv6Network {\n            addr: fd00:1122:3344:100::,\n            prefix: 56,\n        },\n        infra_ip_first: 10.151.2.174,\n        infra_ip_last: 10.151.2.174,\n        ports: [\n            PortConfigV1 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Network {\n                                addr: 0.0.0.0,\n                                prefix: 0,\n                            },\n                        ),\n                        nexthop: 10.151.2.100,\n                        vlan_id: None,\n                    },\n                ],\n                addresses: [\n                    V4(\n                        Ipv4Network {\n                            addr: 10.151.2.174,\n                            prefix: 24,\n                        },\n                    ),\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n    allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.37153224Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1003"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.37153863Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371581361Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371627392Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371640353Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371645203Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1063"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371670893Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371684764Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371700674Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371706084Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1104"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371709674Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371713734Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371738495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.371747365Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372079494Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372289019Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.37230279Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372353611Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372361891Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372523305Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372652108Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.37271254Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 5f617488-2ef8-4cf1-81f7-2b249f5b34c0, rack_id: 240cabb9-b684-4172-8606-d31cd2279cd6, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372839023Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:34794","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 5f617488-2ef8-4cf1-81f7-2b249f5b34c0, rack_id: 240cabb9-b684-4172-8606-d31cd2279cd6, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372939156Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372950106Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.372982307Z","hostname":"sock","pid":1590,"file":"sled-agent/src/server.rs:44"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.373002788Z","hostname":"sock","pid":1590,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.373008248Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.373014388Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.373018248Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.373047418Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.373056049Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.373062449Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.373066159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.376149248Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.38517505Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.395630068Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.406436975Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.415606221Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.42452293Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.433564731Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.442449019Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.451536183Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.460642386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.460676107Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.460680607Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.460685187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.460689388Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.460693798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.47090748Z","hostname":"sock","pid":1590,"sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.470955081Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.470974351Z","hostname":"sock","pid":1590,"sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.471045263Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 135.925954ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.473954448Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f8a4438-05b7-48c6-aa43-d39fa518992c\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f8a4438-05b7-48c6-aa43-d39fa518992c\" }"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.481747798Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_112d7827-193f-456c-94a5-19163d29475e/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.484125269Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.499078563Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"successfully created encrypted zvol: name=\"oxi_112d7827-193f-456c-94a5-19163d29475e/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.506602826Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_112d7827-193f-456c-94a5-19163d29475e/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.506647897Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.511703667Z","hostname":"sock","pid":1590,"sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.511711227Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"}
{"msg":"Ensuring dataset oxi_112d7827-193f-456c-94a5-19163d29475e/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.511721167Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.581923289Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.610234995Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 107.535102ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.610861261Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ce9370f9-ff03-43f8-81d1-33ed389a2d63\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ce9370f9-ff03-43f8-81d1-33ed389a2d63\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.719939031Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 110.970552ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.720402152Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26401e8b-09e1-4e0b-b8ec-b5f724359197\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"26401e8b-09e1-4e0b-b8ec-b5f724359197\" }"}
{"msg":"Mounting oxi_112d7827-193f-456c-94a5-19163d29475e/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.801502964Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.815173074Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.815238416Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.815251266Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.815266097Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.832727645Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 95.368922ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.833283949Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2eab517e-6bc3-4bb9-87ca-3f02aaae61e2\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2eab517e-6bc3-4bb9-87ca-3f02aaae61e2\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.929679663Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 286.030246ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:04.93036345Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6e8b1c31-30bc-4c63-8b1e-1c6034b901f0\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6e8b1c31-30bc-4c63-8b1e-1c6034b901f0\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.217642632Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 358.09952ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.21835982Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"17c2930c-2214-4797-9a39-7c5ba3edb47b\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"17c2930c-2214-4797-9a39-7c5ba3edb47b\" }"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.567572302Z","hostname":"sock","pid":1590,"sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.568393243Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.568424133Z","hostname":"sock","pid":1590,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.568510486Z","hostname":"sock","pid":1590,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:51153","producer_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","route":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:05.576151102Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"231.015917ms"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.579912478Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.622583373Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.622638164Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.622724607Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.622732327Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.623473616Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.623501517Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.623535538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.623545738Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.623852636Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.624080201Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.624100452Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.624106662Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.624154293Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:34794","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.848443789Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.848524561Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.848738047Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.848832119Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.84885965Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.8488674Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.84888386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:05.850014009Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"506.261316ms"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:05.853878378Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.881330463Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54427"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.881531188Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.88160859Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"c26931b0-b45a-44c4-b668-265879ba7244","remote_addr":"[fd00:1122:3344:101::1]:54427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":151,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.911136268Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47178"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.911250441Z","hostname":"sock","pid":1590,"uri":"/sled-role","method":"GET","req_id":"d0494a80-b9c0-4032-b7aa-519d88661755","remote_addr":"[fd00:1122:3344:101::1]:47178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.912322568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.912361939Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.91276941Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.912981565Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: ea952c80-c23a-4ee1-bd80-680243d16d24, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool) }, OmicronPhysicalDiskConfig { id: 14f66ddc-1eb5-48be-b89b-a8fa9533ecf1, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool) }, OmicronPhysicalDiskConfig { id: 725c27a8-a484-4e46-8501-1dfe7e2922bc, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool) }, OmicronPhysicalDiskConfig { id: 88a38c3a-e1c0-4044-94de-a3eb481ddab6, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool) }, OmicronPhysicalDiskConfig { id: efc9f2a3-35bf-41bc-a0ef-d41e2887e1f6, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool) }, OmicronPhysicalDiskConfig { id: 71dc3281-b536-40e6-adbc-653e228e1156, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: ae2f89f7-35c8-4c05-940c-cdee6d8f8a12 (zpool) }, OmicronPhysicalDiskConfig { id: f3ab4730-e13b-472f-b5bc-86c0264d5763, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 42b2209d-f16a-4fca-a84d-54ecc9f13877 (zpool) }, OmicronPhysicalDiskConfig { id: 5cb4d019-d265-4039-9f26-1b59e0c7533d, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 95ce52af-b115-45d4-acd1-3598b0bafa5d (zpool) }, OmicronPhysicalDiskConfig { id: b7371311-29d9-4467-886d-e4e8f1bc545e, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: ad348a98-27df-494e-af4b-560a8a421d64 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.941908268Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:347"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.942075982Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55076"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: ea952c80-c23a-4ee1-bd80-680243d16d24, pool_id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 14f66ddc-1eb5-48be-b89b-a8fa9533ecf1, pool_id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 725c27a8-a484-4e46-8501-1dfe7e2922bc, pool_id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: 88a38c3a-e1c0-4044-94de-a3eb481ddab6, pool_id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: efc9f2a3-35bf-41bc-a0ef-d41e2887e1f6, pool_id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: 71dc3281-b536-40e6-adbc-653e228e1156, pool_id: ae2f89f7-35c8-4c05-940c-cdee6d8f8a12 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: f3ab4730-e13b-472f-b5bc-86c0264d5763, pool_id: 42b2209d-f16a-4fca-a84d-54ecc9f13877 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: 5cb4d019-d265-4039-9f26-1b59e0c7533d, pool_id: 95ce52af-b115-45d4-acd1-3598b0bafa5d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: b7371311-29d9-4467-886d-e4e8f1bc545e, pool_id: ad348a98-27df-494e-af4b-560a8a421d64 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.9423783Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.94238758Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.94239819Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.94240541Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.942431471Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.942439001Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.942447391Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:05.942459612Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:06.101245986Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:06.101285497Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool)"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:06.399949891Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"511.254745ms"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:06.810582269Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:07.00277109Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.568255936s"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.236319813Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.474211098Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.474264219Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.474275079Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.4742853Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.47431131Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.474422283Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.474429603Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:07.689669827Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:07.881102879Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:08.401074052Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:08.623239313Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 40.41434ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:08.845952238Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:08.847754234Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.404201184s"}
{"msg":"Automatically destroying dataset: oxp_c919f4ac-100e-4c1e-bd24-76220f4848fc/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:08.853410409Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 120.201514ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:09.531875999Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:09.766678165Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:09.766720996Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 281.145481ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:09.982167474Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 318.439252ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:10.649214011Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:10.85141696Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 610.081334ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:11.464679967Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:11.890959336Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:11.891133841Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Looking for unencrypted datasets in oxp_c919f4ac-100e-4c1e-bd24-76220f4848fc","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:11.908768603Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:12.289953505Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:12.290017166Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:12.290021826Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:12.290026646Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:12.290029756Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 850.330886ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:12.495672073Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:12.505423044Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:12.505436344Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9d13b91d-5969-417a-bfea-ad89852618b9 (zpool)"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:12.903982862Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:14.085564031Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:14.085668324Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"9.172892393s"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:14.08590504Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:14.309641801Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:14.784303812Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:14.784434036Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:14.785094902Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:14.785126523Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:14.785135923Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:14.785303178Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:14.785317288Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:15.526780475Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:15.527179795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:15.971033724Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:16.182240784Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Automatically destroying dataset: oxp_9d13b91d-5969-417a-bfea-ad89852618b9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.187185321Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.374475487Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"}
{"msg":"Looking for unencrypted datasets in oxp_9d13b91d-5969-417a-bfea-ad89852618b9","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.500985114Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.508815775Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.508856426Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.508864326Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.508876876Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.508911087Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.51916773Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.519193611Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"39005e7e-189e-49db-8463-a206621e7a54 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.612101115Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.619981937Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.620024939Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.620052279Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.62006963Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.62008176Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.620220044Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.620229394Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:16.623942769Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.656852853Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_39005e7e-189e-49db-8463-a206621e7a54/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.664404037Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_39005e7e-189e-49db-8463-a206621e7a54","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.789539738Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.797703468Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.797739999Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.797748499Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.79777497Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.79778295Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.807785407Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.807815478Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"a7771067-edf5-47c9-a972-69320dc9ee62 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.899468839Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.907212528Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.907240119Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.907250579Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.907318881Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.907328711Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.907439844Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.907445714Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.94079764Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_a7771067-edf5-47c9-a972-69320dc9ee62/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:16.94859657Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_a7771067-edf5-47c9-a972-69320dc9ee62","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.077743264Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.086509038Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.08655972Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.08656803Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.08657764Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.08658406Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.096771252Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.096795353Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"096e5012-bddf-482f-8ad5-65f3b0930676 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.192981091Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.201057058Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.20112418Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.20113379Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.201153Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.201162361Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.201288834Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.201300384Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.237859053Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_096e5012-bddf-482f-8ad5-65f3b0930676/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.246089354Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_096e5012-bddf-482f-8ad5-65f3b0930676","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.372034396Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.380138904Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.380180355Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.380186685Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.380191885Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.380196225Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.390213832Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.390237833Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ae2f89f7-35c8-4c05-940c-cdee6d8f8a12 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.484126712Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: ae2f89f7-35c8-4c05-940c-cdee6d8f8a12 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.492067756Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.492111207Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.492117767Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.492158998Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.492170208Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.492332373Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.492342173Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.527949227Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_ae2f89f7-35c8-4c05-940c-cdee6d8f8a12/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.535690425Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:17.625928921Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Looking for unencrypted datasets in oxp_ae2f89f7-35c8-4c05-940c-cdee6d8f8a12","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.664662375Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.672524347Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.672561868Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.672569568Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.672575298Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.672579228Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.682541474Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.682562784Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"42b2209d-f16a-4fca-a84d-54ecc9f13877 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.77514798Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 42b2209d-f16a-4fca-a84d-54ecc9f13877 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.782789626Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.782816617Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.782822057Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.782848208Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.782856968Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.783020582Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.783029692Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.815710291Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_42b2209d-f16a-4fca-a84d-54ecc9f13877/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.822500845Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_42b2209d-f16a-4fca-a84d-54ecc9f13877","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.941471658Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.949117174Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.949145605Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.949152766Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.949158136Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.949164226Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.960534747Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:17.960557308Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"95ce52af-b115-45d4-acd1-3598b0bafa5d (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.057376032Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 95ce52af-b115-45d4-acd1-3598b0bafa5d (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.065193873Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.065226593Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.065233104Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.065269315Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.065278165Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.065442739Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.065452519Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:18.086580152Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.102042198Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_95ce52af-b115-45d4-acd1-3598b0bafa5d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.109837898Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_95ce52af-b115-45d4-acd1-3598b0bafa5d","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.245147341Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.253350211Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.253396082Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.253416093Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.253423893Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.253430103Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.264892227Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.264919488Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ad348a98-27df-494e-af4b-560a8a421d64 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.366024612Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: ad348a98-27df-494e-af4b-560a8a421d64 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.373374001Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.373404192Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.373411622Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.373464454Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.373492634Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.373565376Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.373587077Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.408418591Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_ad348a98-27df-494e-af4b-560a8a421d64/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.416766315Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_ad348a98-27df-494e-af4b-560a8a421d64","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.544479732Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.552556839Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.5525947Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.552617631Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.552805616Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 112d7827-193f-456c-94a5-19163d29475e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e21e7560-2c98-46a4-bdda-f772e50ec2a3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: ae2f89f7-35c8-4c05-940c-cdee6d8f8a12 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 42b2209d-f16a-4fca-a84d-54ecc9f13877 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 95ce52af-b115-45d4-acd1-3598b0bafa5d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: ad348a98-27df-494e-af4b-560a8a421d64 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.552922609Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.553309779Z","hostname":"sock","pid":1590,"uri":"/omicron-physical-disks","method":"PUT","req_id":"13dc1362-9678-457b-b160-6a88f80da4fc","remote_addr":"[fd00:1122:3344:101::1]:55076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":12611121,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.583696578Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.583886343Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:36652"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.584118219Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.584128009Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:18.628410896Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.636466143Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.636476543Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.687848181Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.687881612Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.738551632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.738584183Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.738593213Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.738661925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.786161624Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.786198745Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.786204565Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.841388791Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.84523404Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.849085869Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.860774999Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.865381207Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:18.86977504Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:39:19.630746377Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Profile for oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:2::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:2::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:2::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:2::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:20.386032238Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:20.38610808Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"}
{"msg":"Profile for oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:3::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:3::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:3::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:3::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:20.86387213Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:20.863968133Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:20.867892533Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"}
{"msg":"Profile for oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:1::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:1::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:1::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:1::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:21.349784369Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:21.34982435Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 63.267647ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.366430636Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:21.366478657Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:21.366562309Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 136.481809ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.439288826Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 58.507704ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.477891076Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 32.318312ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.48583967Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 97.637734ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.526255207Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 114.550682ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.550577451Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 125.198734ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.62535501Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 195.593066ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.646527374Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 142.93638ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.699060952Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 426.597544ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.764974963Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 265.438314ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.857040246Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 501.804692ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:21.873275152Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 641.34092ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:22.13859202Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 513.18259ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:22.204545543Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 488.920041ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:22.383459124Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 562.065687ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:22.727579175Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 566.551012ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:22.789091593Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 823.968007ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:22.881394702Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 738.075373ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:23.299802748Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 912.349393ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:23.366284754Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:23.62332547Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"}
{"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:23.623348981Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 781.73021ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:23.714597153Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 1.483623306s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:24.046026907Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe7e:e4da","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:24.173750144Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"}
{"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:24.179285806Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 1.024174813s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:24.28619471Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:24.303873043Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 764.926745ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:24.505234501Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:24.559485933Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:24.763116968Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:25.07377075Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"}
{"msg":"wait for service Some(\"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e\") failed: Property not found. retry in 1.149097928s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:25.280245358Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:25.284093157Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"}
{"msg":"wait for service Some(\"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07\") failed: Property not found. retry in 1.264373424s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:25.319847954Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:25.472974874Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"}
{"msg":"wait for service Some(\"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c\") failed: Property not found. retry in 1.221625926s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:25.538344341Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:25.686965145Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:25.878236294Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:25.889041711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:25.889077542Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:25.889095952Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.446158847Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.45291678Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.614795744Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.72804056Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.753979396Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.78258223Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_12f3cc35-fd2a-411b-bc60-c7832113465e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.782647302Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_baf23778-f6a3-4d4f-931a-1389406eba07"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.782655882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_063ecfd8-b3e0-46f6-98ae-94ccfb48c08c"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.782739504Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.783123554Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.783474913Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"2b1f0bdd-e4c1-46e4-b279-235cdffd3c71","remote_addr":"[fd00:1122:3344:101::1]:36652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8199517,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.785276369Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.861181547Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:26.985187358Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.016993465Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.0483777Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.048460302Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.04877262Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"3"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.04878379Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.2.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122677937Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122746468Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122756009Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122763659Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122768179Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122774179Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122780049Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122785619Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.12280009Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.12280771Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.12281353Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.12281963Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.12282576Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122831411Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122838511Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122844181Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122851351Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122855511Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122861081Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122866992Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122879352Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122883592Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.122887502Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"failed to write DNS configuration (will retry in 56.875444ms)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:27.225318441Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:568","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.300278894Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.347306101Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.391533326Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:584"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.391546986Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.423767433Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.424495252Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.424500672Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.452582222Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.452723066Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49117"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.452950952Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.452958812Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.473821888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.473867459Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.493478192Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.493538213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.51288587Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.512945792Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.513134587Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.513242259Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.525429272Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.556553261Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.557162476Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.557169046Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.60135419Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 240, 111]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.601387111Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.601914405Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.601925075Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.602837398Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.604733217Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.628866886Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:27.646450917Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/chrony-setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value=\"fd00:1122:3344:100::/56\"/>\n        <propval type=\"boolean\" name=\"boundary\" value=\"true\"/>\n        <propval type=\"astring\" name=\"server\" value=\"0.pool.ntp.org\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.3.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.3.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:28.297113184Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:28.297275698Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405\") failed: Property not found. retry in 33.604941ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:28.447462522Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405\") failed: Property not found. retry in 110.177972ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:28.488808343Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405\") failed: Property not found. retry in 218.044964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:28.628701473Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405\") failed: Property not found. retry in 401.603672ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:28.858510091Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405\") failed: Property not found. retry in 1.058527268s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:29.268507371Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405\") failed: Property not found. retry in 1.46509076s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:30.336462826Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:30.411727377Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34210"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:30.411874611Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b1840452-6b7d-4f1d-8778-f297864f0b3f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34210","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":59,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405\") failed: Property not found. retry in 1.3029849s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:31.811754784Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405\") failed: Property not found. retry in 549.981005ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:33.123763281Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.695739519Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.695823431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.696124018Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.696407846Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a5c899b6-31b8-4786-b0cd-25718915ef1b","remote_addr":"[fd00:1122:3344:101::1]:49117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":6243642,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.69854069Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:625"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.728356085Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.728506599Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49560"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.738396463Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.738556577Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"c6e84882-134c-406d-b8b0-d271c161ffdb","remote_addr":"[fd00:1122:3344:101::1]:49560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9903,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:33.741336889Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.911609768Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:33.911706061Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53449"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:37.920314646Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:37.920419978Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"838908cb-2732-40f2-a97f-a994a83a6785","remote_addr":"[fd00:1122:3344:101::1]:53449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008631,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:37.923470437Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:38.023140464Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:38.023260207Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:36070"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:38.031933569Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:38.032059313Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"53c8a11b-98a3-4b0b-91f4-d29650c14bcf","remote_addr":"[fd00:1122:3344:101::1]:36070","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8628,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:38.033948101Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:38.150686167Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:38.150784989Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52763"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_2d5804f9-5e88-4859-b528-dca669659405': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.159657691Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.159744073Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"6552caf7-80ce-4f42-b510-71119911c97e","remote_addr":"[fd00:1122:3344:101::1]:52763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008847,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:42.161431256Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.305532534Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.305658647Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61465"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.315225963Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"cd16ca93-f712-4b65-b699-6be7c676cb7e","remote_addr":"[fd00:1122:3344:101::1]:61465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9401,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.317696006Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:42.317701656Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.624038098Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.624167961Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.63233507Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"ffa7510d-1a0c-4e2e-b04a-b89c449c2bab","remote_addr":"[fd00:1122:3344:101::1]:64381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8030,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.63505275Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:42.635059251Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.892770854Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.892884547Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54951"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.902265388Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"b86bc6d1-8d2c-4028-b382-97fb76a519da","remote_addr":"[fd00:1122:3344:101::1]:54951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9237,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:42.904591027Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:42.904600307Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:43.221341615Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:43.221428957Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:33410"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:43.229302149Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"b3f139a2-1e30-4741-9869-8070cae7ee90","remote_addr":"[fd00:1122:3344:101::1]:33410","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7768,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:43.230746076Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:43.230752406Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:43.29950949Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"309.628126ms"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:43.792534172Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:43.792643045Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49239"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:43.80102579Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"04a2c1bf-642d-482c-a832-0b26ef176038","remote_addr":"[fd00:1122:3344:101::1]:49239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8248,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:43.802818806Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:43.802827426Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.092737325Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.092838288Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42874"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.100642768Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"384d00f8-79d6-4880-a3eb-50b82b07c61f","remote_addr":"[fd00:1122:3344:101::1]:42874","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7686,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.102161277Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.102165547Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.476644967Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.47675449Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44946"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.485758741Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"455fa1f2-40d2-4509-a227-fa296c709350","remote_addr":"[fd00:1122:3344:101::1]:44946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8871,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.488939243Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.488948933Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.265043781Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.265156474Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61123"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.268433718Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.289185011Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"a4a6f110-0e2a-4999-9702-8fb63ef48ff2","remote_addr":"[fd00:1122:3344:101::1]:61123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":23866,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 396892370, ip_addr: 23.168.24.210, stratum: 3, ref_time: 1715679584.1683795, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.290999137Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.291006588Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1216"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.319475248Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.319606322Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59710"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.319828607Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.319838708Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.339931363Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.339972264Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.359345701Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.359383142Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.378465782Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.378502623Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.431224796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.431262437Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.485750985Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.485789456Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.538108969Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.53814849Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.590087303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.590134754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.641925843Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.641978024Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.642260071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.642391435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.674991451Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.675034092Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.675043563Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.675053053Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.675060803Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.675190026Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"337e361e-741f-4088-89b0-edb3707fc77f","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":355521,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.675409512Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"129.188ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"337e361e-741f-4088-89b0-edb3707fc77f\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"337e361e-741f-4088-89b0-edb3707fc77f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.805549472Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.806034184Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.806049585Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.825169535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.825207956Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.843187827Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.843222678Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.861061736Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.861097257Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.879018507Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.879044738Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.896644349Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.89667904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.914657001Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.914693753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.933001012Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.933037943Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.951451976Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.951489307Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.951679642Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.951805325Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.984496024Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.984544955Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.984554905Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.984564206Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.984574876Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:44.984655848Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"383366e7-f2c4-4d98-acf5-3113ee71212d","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized)]","latency_us":178861,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:44.984830902Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"79.06442ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"383366e7-f2c4-4d98-acf5-3113ee71212d\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"383366e7-f2c4-4d98-acf5-3113ee71212d\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.06500364Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.065369359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.065388869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.085475435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.085507436Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.102213654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.102257856Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.117510967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.117540378Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.137088689Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.13711845Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.157050172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.157097393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.174718295Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.174751616Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.194156663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.194196354Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.209681071Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.209723563Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.209935568Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.210062581Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.239657641Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.239698552Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.239706472Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.239712842Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.239718122Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.239806975Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a6e957b4-2421-4bff-82a3-b36e8ae9fd5f","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized)]","latency_us":174637,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.239977779Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"193.955876ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a6e957b4-2421-4bff-82a3-b36e8ae9fd5f\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a6e957b4-2421-4bff-82a3-b36e8ae9fd5f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.434268465Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.434624224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.434638634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.456062254Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.456106445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.473787169Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.47383904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.491927394Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.491970786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.511943918Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.511976169Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.529229672Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.529258523Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.547407298Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.547452529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.566882138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.566913339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.586812829Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.58684873Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.587071676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.58720195Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.622031533Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.622072044Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.622079514Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.622088985Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.622098465Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.622238369Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"cfb226ee-3cb4-4e20-8e39-2b76e9040778","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":187796,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:45.622439204Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"98.79746ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cfb226ee-3cb4-4e20-8e39-2b76e9040778\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cfb226ee-3cb4-4e20-8e39-2b76e9040778\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.722790949Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.723136968Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.723151838Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.743946572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.743993263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.763422861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.763466043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.781638659Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.7816741Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.801425377Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.801464768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.820849635Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.820885826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.840247013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.840283534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.859776824Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.859812125Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.879317256Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.879359857Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.879522481Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:45.879648724Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.004369022Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.011725781Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.011739011Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.011745441Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.011752791Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.011772602Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.011909065Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5e135159-0c10-41d2-bdf8-a297d1adbd85","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized)]","latency_us":190943,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.012128341Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"307.235663ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e135159-0c10-41d2-bdf8-a297d1adbd85\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5e135159-0c10-41d2-bdf8-a297d1adbd85\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.320548565Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.321702074Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.321713885Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.342598441Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.342628891Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.36205677Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.362083041Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.381863138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.381898619Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.40101512Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.40104548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.419948995Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.419993796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.438344178Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.438369858Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.458180716Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.458222348Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.477304557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.477339448Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.477530243Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.477653496Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.511057103Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.511099354Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.511105785Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.511113815Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.511120215Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:46.511233668Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"3806c413-1a73-4cde-b766-57a9308e24d2","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized)]","latency_us":190488,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:46.511406782Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"354.440447ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3806c413-1a73-4cde-b766-57a9308e24d2\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3806c413-1a73-4cde-b766-57a9308e24d2\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.268588743Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.268911931Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.268924861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.272142154Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.289976372Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.290029783Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.308562029Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.30859759Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.32613979Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.326180861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.34602842Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.346076121Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.364315389Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.364352341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.383734978Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.383770689Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.401403701Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.401440692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.419476345Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.419515246Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.41969467Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.419807353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.45203648Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.452078562Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.452088332Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.452096052Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.452103682Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.452239186Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c8900546-87fc-42d0-877b-ca0b52554360","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":183474,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.452483902Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"333.901955ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c8900546-87fc-42d0-877b-ca0b52554360\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c8900546-87fc-42d0-877b-ca0b52554360\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.787895538Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.788381401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.788402122Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.808284692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.808318283Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.826397886Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.826425047Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.846607625Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.846636146Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.864927245Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.864952066Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.882807774Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.882831055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.902827508Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.902857119Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.920479491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.920502932Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.938129064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.938165325Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.9383653Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.938492153Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.970969816Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.971006648Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.971014118Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.971019418Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.971028038Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:47.971177852Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"dad4e0da-311c-43e8-8c6b-548b20247a17","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":183022,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:47.971913611Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"415.247873ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dad4e0da-311c-43e8-8c6b-548b20247a17\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dad4e0da-311c-43e8-8c6b-548b20247a17\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.388358467Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.389716872Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.389774864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.410428094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.410460135Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.429595436Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.429629857Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.449184768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.449224269Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.468302789Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.46833187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.488078937Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.488110847Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.506646833Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.506674514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.526771539Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.52679983Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.545211972Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.545282034Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.545546041Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.545680475Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:48.579606235Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:48.579651956Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:48.579660507Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:48.579670647Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:48.579678487Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.579769179Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"070ffbe9-e787-42e1-a9df-2dd78ed05a13","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized)]","latency_us":191224,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:48.579937054Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"362.34392ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"070ffbe9-e787-42e1-a9df-2dd78ed05a13\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"070ffbe9-e787-42e1-a9df-2dd78ed05a13\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.943339379Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.943697868Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.943710708Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.962968632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.963021694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.981139249Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.98117883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.999382777Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:48.999435898Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.019720469Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.01976861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.037429673Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.037474204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.057011886Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.057046807Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.075047188Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.07509127Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.09458126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.094647971Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.094826686Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.09496584Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.128453299Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.128513971Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.128526121Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.128536161Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.128565622Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.128650464Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"47c55e30-c9ba-47da-8edc-61414da7743b","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":185138,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.128835239Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"520.274602ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"47c55e30-c9ba-47da-8edc-61414da7743b\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"47c55e30-c9ba-47da-8edc-61414da7743b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.650236129Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.650580378Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.650597908Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.670699164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.670748205Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.689477765Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.689531526Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.707811276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.707861357Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.726730441Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.726775202Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.745817191Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.745859562Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.763407872Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.763453473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.782256376Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.782313007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.800861663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.800904384Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.80114046Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.801290584Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.832974287Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.833019998Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.833030609Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.833040559Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.833050749Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:49.833183033Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"972054ce-b943-415b-b73f-c0256e8fa3fd","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":182775,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:49.833404708Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"607.373731ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"972054ce-b943-415b-b73f-c0256e8fa3fd\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"972054ce-b943-415b-b73f-c0256e8fa3fd\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.441909364Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.442243152Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.442265613Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.463007925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.463054096Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.481388797Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.481437048Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.499511882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.499558773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.519297309Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.51933217Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.538109932Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.538148153Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.557175221Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.557208942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.575747418Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.575789989Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.594280744Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.594338405Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.594549301Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.594688524Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:50.629831996Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:50.629897628Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:50.629908438Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:50.629918518Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:50.629928479Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:50.630062512Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"71d47320-6b19-4d98-9432-b8ee4c4448a3","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":187994,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:50.630276528Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.099324277s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"71d47320-6b19-4d98-9432-b8ee4c4448a3\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"71d47320-6b19-4d98-9432-b8ee4c4448a3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.730838088Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.731210608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.731227108Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.751078898Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.751141869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.769746407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.769797368Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.787150523Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.787201145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.805832723Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.805889594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.824258996Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.824296467Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.842407781Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.842461033Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.860900746Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.860946887Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.878953779Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.879015591Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.879268287Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.879419741Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:51.91249359Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:51.912524261Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:51.912532171Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:51.912539921Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:51.912547961Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:51.912698775Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"309208b4-eecd-43a4-8f61-c5cfe1944eca","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized)]","latency_us":181708,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:51.91287636Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.225792066s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"309208b4-eecd-43a4-8f61-c5cfe1944eca\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"309208b4-eecd-43a4-8f61-c5cfe1944eca\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.140098152Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.140447601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.140459631Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.164233191Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.164264012Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.185555978Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.185592819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.205087389Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.20511678Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.22419618Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.224230621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.244126291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.244154762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.26317346Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.263209551Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.282319161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.282352642Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.30330028Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.303345941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.303555826Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.30370236Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:53.339224471Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:53.339291923Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:53.339305364Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:53.339314724Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:53.339321724Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.339437307Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"060d6c7c-8f46-4609-b4d9-1b600659a1dd","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized)]","latency_us":199188,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:53.339650142Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"559.779472ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"060d6c7c-8f46-4609-b4d9-1b600659a1dd\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"060d6c7c-8f46-4609-b4d9-1b600659a1dd\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.901192232Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.901745276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.901770687Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.919861951Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.919914542Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.936530649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.93657272Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.953901464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.953943516Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.972781099Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.97281858Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.992732901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:53.992773762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.010856086Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.010889427Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.029503224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.029540885Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.049485677Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.049532238Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.049743624Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.049872797Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:54.083558712Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:54.083611983Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:54.083620333Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:54.083630013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:54.083637404Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:54.083762467Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e811aeeb-1ebd-4d41-8ffd-27778c644e80","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":182269,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:54.084007553Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.366254962s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e811aeeb-1ebd-4d41-8ffd-27778c644e80\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e811aeeb-1ebd-4d41-8ffd-27778c644e80\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.451548006Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.451945576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.451963897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.476470595Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.476525817Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.49769708Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.497740771Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.517365935Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.517418326Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.535502421Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.535535551Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.555285848Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.555320089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.572220893Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.572269014Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.592348779Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.59239552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.609613202Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.609661833Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.609868179Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.609977232Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:55.643189434Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:55.643239935Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:55.643250205Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:55.643258366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:55.643268526Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:55.643392829Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"098a6d24-c3b9-4eb2-b13f-07fa58e2dd8a","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":191667,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:55.643620755Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.533759495s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"098a6d24-c3b9-4eb2-b13f-07fa58e2dd8a\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"098a6d24-c3b9-4eb2-b13f-07fa58e2dd8a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.178226184Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.179102977Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.179115187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.19910397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.199143061Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.217849151Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.217879082Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.236780357Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.236827208Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.255439145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.255476226Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.274554796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.274597327Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.293063341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.293094532Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.3109538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.310981211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.328797428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.328838179Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.329027294Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.329135947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:57.362449842Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:57.362486343Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:57.362495303Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:57.362512953Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:57.362522284Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:39:57.362617966Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a9cd4a5c-782a-47d8-bb23-7bc8faeda3d3","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized)]","latency_us":184252,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:57.362803211Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"2.741833845s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a9cd4a5c-782a-47d8-bb23-7bc8faeda3d3\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:39:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a9cd4a5c-782a-47d8-bb23-7bc8faeda3d3\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:39:58.361614131Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"598.802377ms"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.10585625Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.1062503Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.106265301Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.127943287Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.127992078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.145684992Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.145712473Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.16547555Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37350"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.165615404Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f4252f63-f348-40f5-a94f-a7204b682da1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37350","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":81,"response_code":"200"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.167064151Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.167072261Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.186949191Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.186993392Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.206860742Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.206917703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.227916293Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.227957363Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.248780018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.248816579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.268570956Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.268624287Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.268858893Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.269005157Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:00.302462595Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:00.302506376Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:00.302516207Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:00.302525527Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:00.302543817Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:00.30265764Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1fb2a78b-29a3-4ab8-8db7-9688bbfb0c82","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized)]","latency_us":196633,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:00.302846925Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.198845655s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1fb2a78b-29a3-4ab8-8db7-9688bbfb0c82\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1fb2a78b-29a3-4ab8-8db7-9688bbfb0c82\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.502313365Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.502539211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.502550491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.523003546Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.523065477Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.541416798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.541454059Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.559246756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.559305258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.57813197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.578164921Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.595727622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.595765323Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.613850797Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.613889258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.63537263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.635424861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.653598587Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.653656179Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.653847453Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.653984737Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:01.687099706Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:01.687137547Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:01.687146277Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:01.687154567Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:01.687162698Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:01.687290501Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"219efba1-916a-491c-a88b-723632b53a56","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized)]","latency_us":184879,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:01.687485516Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"3.194958201s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"219efba1-916a-491c-a88b-723632b53a56\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"219efba1-916a-491c-a88b-723632b53a56\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.883472208Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.883750506Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.883760016Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.906419107Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.906470398Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.925879807Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.925931378Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.94432092Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.944364351Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.963953374Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.963995925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.983602308Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:04.98366593Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.002154394Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.002187005Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.021341086Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.021383807Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.041499244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.041563355Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.041838612Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.041983556Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:05.074524731Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:05.074582723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:05.074593053Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:05.074600723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:05.074610323Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:05.074756097Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e3c293a2-4aa0-48a8-819f-278cf45347c9","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized)]","latency_us":191137,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:05.074998433Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.911824662s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e3c293a2-4aa0-48a8-819f-278cf45347c9\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e3c293a2-4aa0-48a8-819f-278cf45347c9\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:06.987259664Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:06.987826018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:06.987842899Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.009196577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.009232128Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.033008748Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.033048359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.053648497Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.053683168Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.072883391Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.072918482Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.094428904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.094452264Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.112894728Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.112935809Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.132663315Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.132699756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.151502788Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.15154952Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.151770655Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.151918419Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:07.185480551Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:07.185508681Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:07.185518172Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:07.185527862Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:07.185537192Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:07.185665935Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ea10be0f-6f20-493d-96af-e16469a9a5f9","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized)]","latency_us":198104,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:07.18585111Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"2.890947956s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea10be0f-6f20-493d-96af-e16469a9a5f9\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ea10be0f-6f20-493d-96af-e16469a9a5f9\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.077621486Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.078021846Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.078036776Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.101843407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.101880438Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.12142312Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.12145546Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.141268289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.14130296Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.158984574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.159015484Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.178909685Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.178938006Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.199304428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.199340439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.21808173Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.218109911Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.236225726Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.236291947Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.236504033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.236628046Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:10.270227078Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:10.270263709Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:10.270270969Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:10.27027797Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:10.27028497Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:10.270422173Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"99917842-883b-4d3f-af74-cbbff0c196ee","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized)]","latency_us":192596,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:10.270610258Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"3.148046513s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"99917842-883b-4d3f-af74-cbbff0c196ee\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"99917842-883b-4d3f-af74-cbbff0c196ee\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.420066706Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.420453376Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.420469947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.443491947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.443561849Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.463146692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.463187713Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.484050128Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.484089009Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.501862025Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.501900246Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.522629668Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.522671979Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.542135789Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.542169359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.561196228Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.561240599Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.580119793Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.580156494Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.58038482Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.580511863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:13.612358431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:13.612387541Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:13.612392881Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:13.612399512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:13.612406802Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:13.612519715Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f6cf3d97-b6fc-4189-a7c2-f4566c12a2b4","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":192298,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:13.612705189Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"4.21476287s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f6cf3d97-b6fc-4189-a7c2-f4566c12a2b4\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f6cf3d97-b6fc-4189-a7c2-f4566c12a2b4\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:13.996483477Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.13192185s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.828316286Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.829074555Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.829085865Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.851151012Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.851219024Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.870296853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.870342094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.88693931Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.886982021Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.904236694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.904275495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.922096062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.922143753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.940177266Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.940213147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.957784008Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.957822169Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.975080692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.975140913Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.975340089Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:17.975486873Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:18.006245822Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:18.006281593Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:18.006290213Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:18.006296613Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:18.006305373Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:18.006447337Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5315df72-2f84-4b64-86e9-5f03648923bf","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":178036,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:18.006642782Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"5.11701742s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5315df72-2f84-4b64-86e9-5f03648923bf\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5315df72-2f84-4b64-86e9-5f03648923bf\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.1252252Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.125546538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.125564429Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.14859845Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.148642511Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.167975967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.168012528Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.185144598Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.185203299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.205686795Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.205731446Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.223869592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.223906502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.241850643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.241892164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.259104706Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.259142187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.279079328Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.27912145Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.279359105Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.279499389Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:23.311587092Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:23.311625384Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:23.311633284Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:23.311641074Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:23.311647744Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:23.311782608Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c3cca7c0-7fba-42b3-babc-9eb81adba179","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":186429,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:23.311967642Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"10.717732489s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c3cca7c0-7fba-42b3-babc-9eb81adba179\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c3cca7c0-7fba-42b3-babc-9eb81adba179\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:30.164124575Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.967031486s"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:30.197761689Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50053"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:30.197856481Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"cbcb1a63-91e2-4d3b-aaf4-9dd8af1a1b94","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50053","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.031134517Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.031474385Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.031488495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.054549877Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.054584168Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.073673618Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.073729559Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.091754962Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.091800123Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.111587901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.111639122Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.130789874Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.130833085Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.149227107Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.149264158Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.16689384Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.166935051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.187560521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.187604702Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.187879809Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.188017802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:34.222427815Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:34.222467587Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:34.222475067Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:34.222480977Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:34.222488207Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:34.22260979Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"26bb3c6a-88f6-41e3-b962-18db206e60cd","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized)]","latency_us":191320,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:34.222814055Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"9.730094077s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"26bb3c6a-88f6-41e3-b962-18db206e60cd\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"26bb3c6a-88f6-41e3-b962-18db206e60cd\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:43.953815202Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:43.95410289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:43.95411341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:43.974887803Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:43.974921014Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:43.990335309Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:43.9903654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.010940338Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.010975539Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.030021488Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.030048779Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.050431692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.050458432Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.069007528Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.069046579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.088225452Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.088257042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.106763737Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.106799208Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.106995143Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.107120896Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:44.140592135Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:44.140669957Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:44.140684328Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:44.140693858Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:44.140705538Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:44.140854742Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"0a3ee484-6cf5-4eb0-89dd-15459303c3d0","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized)]","latency_us":186872,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:44.141061147Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"13.157976652s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0a3ee484-6cf5-4eb0-89dd-15459303c3d0\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0a3ee484-6cf5-4eb0-89dd-15459303c3d0\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:47.166672568Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"3.041721484s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.299152627Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.299467936Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.299480606Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.32457053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.324612851Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.346894163Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.346932794Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.367766919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.36782301Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.388010608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.388055239Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.40795517Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.407997081Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.427864141Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.427901442Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.445567035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.445605216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.465353953Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.465398604Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.465672871Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.465805855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:57.497644862Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:57.497676793Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:57.497682363Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:57.497688693Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:57.497695323Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:40:57.497815796Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"276a7612-afb3-4b50-838c-bd2d85272caf","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized)]","latency_us":198568,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:40:57.498016831Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"10.792519149s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"276a7612-afb3-4b50-838c-bd2d85272caf\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:40:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"276a7612-afb3-4b50-838c-bd2d85272caf\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:00.229909875Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37369"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:00.229987767Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"e01878ef-3953-4b77-8a6f-0c1b8233f415","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37369","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":45,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:05.241880087Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"6.265348524s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.290436257Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.290753365Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.290764126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.311900658Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.311927079Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.330006993Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.330028273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.347207054Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.347256506Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.366318175Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.366352936Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.38403317Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.38406058Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.401406865Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.401428356Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.419834308Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.419860109Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.438517218Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.438559559Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.438805465Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.438931559Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:08.47250092Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:08.472551581Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:08.472559642Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:08.472569112Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:08.472578832Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:08.472693055Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8ccdeb9a-35fa-4893-b8ab-55907d91e0bc","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized)]","latency_us":182140,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:08.47290337Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"8.671205912s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8ccdeb9a-35fa-4893-b8ab-55907d91e0bc\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:41:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8ccdeb9a-35fa-4893-b8ab-55907d91e0bc\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.144250328Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.144469004Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.144481045Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.164251642Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.164293543Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.185056356Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.185089147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.204737671Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.204768882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.223302677Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.223336858Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.243427244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.243493736Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.262451452Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.262485213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.280520356Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.280554647Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.297533593Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.297573534Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.297775109Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.297907812Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:17.329347429Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:17.329396311Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:17.329405581Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:17.329423481Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:17.329431301Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:17.329538944Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"8e9cf4c1-c1fe-4959-9f9c-a2b57c45fab8","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized)]","latency_us":185234,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:17.32977009Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"10.902692685s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e9cf4c1-c1fe-4959-9f9c-a2b57c45fab8\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:41:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e9cf4c1-c1fe-4959-9f9c-a2b57c45fab8\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:26.541779545Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"11.623793173s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.232420305Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.233554434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.233566634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.254247085Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.254296436Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.274565536Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.274608558Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.293614835Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.293663347Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.31172701Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.311765511Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.330551013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.330596784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.348697269Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.34873842Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.366572318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.366610479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.384483658Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.384537329Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.384810786Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.384926699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:28.419402044Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:28.419433624Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:28.419440925Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:28.419449025Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:28.419455565Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:28.419586648Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f9dcb85e-4a4b-43ae-bedb-db33d9518b62","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized)]","latency_us":187045,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:28.419806144Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"17.941749623s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9dcb85e-4a4b-43ae-bedb-db33d9518b62\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:41:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9dcb85e-4a4b-43ae-bedb-db33d9518b62\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:30.259437458Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50680"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:30.259580822Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"143d927a-8d4b-4625-9ac7-bbc1834ba489","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50680","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":62,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.359131074Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.359424942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.359437522Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.383201792Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.383239643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.402922958Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.402963649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.422383787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.422425608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.443649773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.443688484Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.463311888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.463346299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.482829419Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.48286577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.501740664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.501787325Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.521981214Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.522025995Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.522315572Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.522468656Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:46.554064897Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:46.554102818Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:46.554110688Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:46.554118768Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:46.554125188Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:41:46.554278532Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"d9eb2727-5112-45e1-a028-bd7786df951b","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized)]","latency_us":195063,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:46.554510838Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"29.184726445s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d9eb2727-5112-45e1-a028-bd7786df951b\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:41:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d9eb2727-5112-45e1-a028-bd7786df951b\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:41:53.197723595Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"23.342354587s"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:00.287949023Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39390"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:00.288031596Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"03f79863-29a2-4b58-a39d-043935c4bd2b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39390","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":43,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.736512074Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.736712109Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.736726339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.759989786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.760056758Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.781008256Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.781058487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.800198688Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.800244939Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.82130094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.821346491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.839452496Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.839493197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.859268965Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.859303675Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.879838112Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.879880194Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.899503117Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.899555958Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.899837136Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.899973919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:15.93506428Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:15.935112741Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:15.935124421Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:15.935136201Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:15.935150172Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:15.935316796Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ec6732ec-7dfd-4721-86bc-38d79e30fbc2","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized)]","latency_us":198746,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:15.935550922Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"42.518667239s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ec6732ec-7dfd-4721-86bc-38d79e30fbc2\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:42:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ec6732ec-7dfd-4721-86bc-38d79e30fbc2\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:30.317872679Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34470"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:30.318000662Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"38cfc161-b4f9-45f2-ae62-090c43c8fef4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34470","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":73,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:31.572441407Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"79.465305238s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.449460844Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.449788073Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.449800573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.470430583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.470487114Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.489874331Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.489936433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.507871844Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.507917055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.527450556Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.527489877Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.543372784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.543415026Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.560699589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.56073723Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.578795174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.578834385Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.597904194Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.597960445Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.598205972Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.598321335Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:58.629716748Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:58.629746391Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:58.629754192Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:58.629760282Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:58.629765282Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:42:58.629865595Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5ea67e25-ff50-4b14-96bf-dc405b86b51e","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\", TimeNotSynchronized), (\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\", TimeNotSynchronized), (\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\", TimeNotSynchronized), (\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\", TimeNotSynchronized), (\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\", TimeNotSynchronized)]","latency_us":180294,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:42:58.630027729Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"41.07522185s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5ea67e25-ff50-4b14-96bf-dc405b86b51e\", \"content-length\": \"124\", \"date\": \"Tue, 14 May 2024 09:42:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5ea67e25-ff50-4b14-96bf-dc405b86b51e\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:00.347942638Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50751"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:00.348074541Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"6a07dc5a-b73b-4c89-a4a2-ffb46c12b6ea","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50751","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":73,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:30.381723343Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39229"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:30.381827846Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"1797c901-1e45-441f-b84b-d5c6c9a1a7b2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39229","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":55,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.705769609Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.706113698Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.706125448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.729635652Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.729682223Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.749319357Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.749354707Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.769355871Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.769394162Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.788225575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.788261226Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.806520845Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.806555435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.824294221Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.824328822Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.844399827Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.844433278Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.862526152Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.862576903Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.862877991Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.862994204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.946277201Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.946332492Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.946338282Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.946345273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:39.946350853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.0432784Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.047384495Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.051752418Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.056084059Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.060545873Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.065663015Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.070328774Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.074742148Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.079387967Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:40.083981075Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService6\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::5\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:42.725294554Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:42.725427787Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService7\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::7\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:42.734156071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:42.734323436Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::3\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::3\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:42.739866678Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:42.740047342Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 51.198061ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:42.867901983Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 60.779964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:42.875270322Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 48.741053ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:42.881301437Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService9\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::6\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:42.884099259Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:42.884232883Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 95.341631ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:42.926758184Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 112.946537ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.045132741Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 113.575542ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.087586131Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 293.318699ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.283612001Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 213.222129ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.354661795Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 113.713364ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.425145493Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 62.827675ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.485209425Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService8\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::4\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::4\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:43.496092334Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:43:43.496246968Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 270.394394ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.557550301Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 86.728952ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.615101558Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 413.429308ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.651769929Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 539.514589ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.669276908Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 191.764577ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.718200223Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 74.125904ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.825850776Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 425.947729ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.846722281Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 118.627387ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:43.978163804Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 326.978848ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:44.374891935Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 865.614814ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:44.447880788Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 592.894659ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:44.587749057Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 647.768735ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:44.694851045Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 290.668197ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:44.762164642Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 580.913001ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:44.911209707Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 471.853883ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:45.084747011Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 1.000199492s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:45.247923018Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 563.797919ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:45.325044537Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 971.838419ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:45.354253316Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 539.984316ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:45.505847727Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 807.315315ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:45.568277149Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 1.031225605s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:45.899138059Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 588.978639ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:46.055397319Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 570.399245ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:46.257512555Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 707.109269ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:46.335309032Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 1.21447934s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:46.384121484Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 1.474333151s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:46.654457022Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 906.117695ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:46.836692467Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 592.75353ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:46.938832538Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 998.199928ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:47.051424768Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 857.500597ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:47.541507594Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 1.181997022s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:47.608092183Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 591.309511ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:47.752610311Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 1.169931843s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:48.061440056Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 729.007643ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:48.140237698Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 1.448501118s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:48.355926023Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 869.438576ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:48.411190712Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 1.462691171s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:48.798640623Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 1.1676527s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:48.878693757Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 1.295929357s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:49.245802088Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 712.427192ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:49.294077147Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 917.053969ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:49.81638697Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 1.359798764s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:50.015829278Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 729.155054ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:50.056468401Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 754.442842ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:50.274808304Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 1.299903021s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:50.551048983Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 565.631567ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:50.742434883Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 620.80952ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:50.796025878Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d\") failed: Property not found. retry in 574.397824ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:51.039218169Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265\") failed: Property not found. retry in 611.700492ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:51.319789299Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529\") failed: Property not found. retry in 1.225653739s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:51.388443251Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf\") failed: Property not found. retry in 1.497475045s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:51.426939799Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 1.201129612s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:51.863159653Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 1.240902912s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:53.075852752Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 688.687194ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:54.327611194Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 1.174529612s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:55.223022221Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 826.103099ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:56.7654023Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 916.551356ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:57.606480424Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 531.10736ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:58.533854781Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 811.847553ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:59.074890575Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 1.239020149s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:43:59.89852054Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:00.423602964Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34793"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:00.424273011Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"e6af8ade-af0f-4485-bfc0-4a3cef400b55","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34793","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":373,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 936.301647ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:01.147625873Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997\") failed: Property not found. retry in 1.362618837s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:02.095905807Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.480251242Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_dd078000-affd-4ab0-8f5f-c717d7b09529"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.480314404Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_4e2ad775-09a8-4891-8164-63c755a82265"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.480326494Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_d13d0066-b539-4ac5-a545-c60023ae5997"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.480335314Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.480448577Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_277c98ec-b455-4e60-8559-6222d185ceaf"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.481371931Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.482868069Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.483981568Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"741caa31-9cad-4039-bdcd-32047e19fac0","remote_addr":"[fd00:1122:3344:101::1]:59710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":23778259,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.5281221Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61446"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.562847182Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:03.930075814Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.628691385Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.628789417Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"62465402-dd29-40c9-87c3-f54329073ee7","remote_addr":"[fd00:1122:3344:101::1]:61446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":11100720,"response_code":"204"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:14.640607191Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"126.131434258s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 12f3cc35-fd2a-411b-bc60-c7832113465e, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: baf23778-f6a3-4d4f-931a-1389406eba07, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 2d5804f9-5e88-4859-b528-dca669659405, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: fd686f38-d008-435a-ab61-a03c14315355, kind: Service { id: 2d5804f9-5e88-4859-b528-dca669659405 }, name: Name(\"ntp-2d5804f9-5e88-4859-b528-dca669659405\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 240, 111])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dd078000-affd-4ab0-8f5f-c717d7b09529, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4e2ad775-09a8-4891-8164-63c755a82265, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 277c98ec-b455-4e60-8559-6222d185ceaf, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: d13d0066-b539-4ac5-a545-c60023ae5997, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7d57f350-6d84-441e-aca9-427498419506, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b5cecdd2-5810-47a4-acef-30ed4d68f165, kind: Service { id: 7d57f350-6d84-441e-aca9-427498419506 }, name: Name(\"external-dns-7d57f350-6d84-441e-aca9-427498419506\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 209, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bd856afa-64af-4f62-8083-7cec6f530e9b, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: c3a0d6a8-690b-4d68-9c53-ea94903a313d, kind: Service { id: bd856afa-64af-4f62-8083-7cec6f530e9b }, name: Name(\"external-dns-bd856afa-64af-4f62-8083-7cec6f530e9b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: b34dcba8-9ce7-4e76-8638-6efaa0e9a52e, kind: Service { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade }, name: Name(\"nexus-1af27ceb-2d94-42b4-abb9-e7054fec8ade\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 74])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d7310b7c-67e1-4338-8414-09460b7646e9, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 0e69104d-a1e3-43ae-9fa7-d0ba3efaba6d, kind: Service { id: d7310b7c-67e1-4338-8414-09460b7646e9 }, name: Name(\"nexus-d7310b7c-67e1-4338-8414-09460b7646e9\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: afa350c1-4cae-457d-b6fa-02b99bddeae1, kind: Service { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3 }, name: Name(\"nexus-10cdd4df-0856-4bcf-af87-e5757529b0c3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 29])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ec8d4748-9593-47c2-9fb9-59f95eb0cf38, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 712a9949-23a6-4a4a-97da-2917d03922ee, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 3e918472-0cb7-49b5-a622-8308a9ca651a, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 43c965bc-d0c5-4d77-8dee-7a6da492a2da, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: beb02584-2e2a-4158-9aa0-87876e6264fa, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f204562-0cc1-49bf-9bd5-796c36691eb5, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External } } } }, OmicronZoneConfig { id: b8889290-9edc-47af-9bea-f3fc3227987f, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External } } } }, OmicronZoneConfig { id: 87eb850e-70d3-4390-9948-908831c0c747, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External } } } }, OmicronZoneConfig { id: 16207ecc-3887-4690-8f5f-8dfc1add1473, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed0566ef-ce2f-4a35-94d5-9861855ca4b5, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ae2f89f7-35c8-4c05-940c-cdee6d8f8a12 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42b2209d-f16a-4fca-a84d-54ecc9f13877 (zpool), kind: External } } } }, OmicronZoneConfig { id: b794d8c1-39ec-4743-9237-aeb9d36e8e47, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 95ce52af-b115-45d4-acd1-3598b0bafa5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ae414c2-10bd-42e6-9eda-6b0420abb343, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ad348a98-27df-494e-af4b-560a8a421d64 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.663676542Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.664197925Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.665117909Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.665126059Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12f3cc35-fd2a-411b-bc60-c7832113465e, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.688611212Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.688662463Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 063ecfd8-b3e0-46f6-98ae-94ccfb48c08c, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.708990465Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.709038906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: baf23778-f6a3-4d4f-931a-1389406eba07, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.729554013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.729599203Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: dd078000-affd-4ab0-8f5f-c717d7b09529, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.749064783Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.749114515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a9c1f9a0-e708-48b3-b1e9-3d9fabafd80d, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.770891903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.770939684Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4e2ad775-09a8-4891-8164-63c755a82265, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.789724586Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.789766468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 277c98ec-b455-4e60-8559-6222d185ceaf, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.810244523Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.810287644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d13d0066-b539-4ac5-a545-c60023ae5997, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7d57f350-6d84-441e-aca9-427498419506, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.829505697Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.829547339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7d57f350-6d84-441e-aca9-427498419506, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd856afa-64af-4f62-8083-7cec6f530e9b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.943824581Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:14.943876382Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd856afa-64af-4f62-8083-7cec6f530e9b, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 712a9949-23a6-4a4a-97da-2917d03922ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.049414131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.049459752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 712a9949-23a6-4a4a-97da-2917d03922ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: beb02584-2e2a-4158-9aa0-87876e6264fa, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.140002355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.140054097Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: beb02584-2e2a-4158-9aa0-87876e6264fa, dataset_name: DatasetName { pool_name: ZpoolName { id: c919f4ac-100e-4c1e-bd24-76220f4848fc (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f204562-0cc1-49bf-9bd5-796c36691eb5, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.240417232Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.240468133Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1f204562-0cc1-49bf-9bd5-796c36691eb5, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d13b91d-5969-417a-bfea-ad89852618b9 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8889290-9edc-47af-9bea-f3fc3227987f, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.334882566Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.334930638Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8889290-9edc-47af-9bea-f3fc3227987f, dataset_name: DatasetName { pool_name: ZpoolName { id: 39005e7e-189e-49db-8463-a206621e7a54 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 87eb850e-70d3-4390-9948-908831c0c747, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.429125485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.429170616Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 87eb850e-70d3-4390-9948-908831c0c747, dataset_name: DatasetName { pool_name: ZpoolName { id: a7771067-edf5-47c9-a972-69320dc9ee62 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 16207ecc-3887-4690-8f5f-8dfc1add1473, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.52479413Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.524864492Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 16207ecc-3887-4690-8f5f-8dfc1add1473, dataset_name: DatasetName { pool_name: ZpoolName { id: 096e5012-bddf-482f-8ad5-65f3b0930676 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed0566ef-ce2f-4a35-94d5-9861855ca4b5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae2f89f7-35c8-4c05-940c-cdee6d8f8a12 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.61208746Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.612132081Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed0566ef-ce2f-4a35-94d5-9861855ca4b5, dataset_name: DatasetName { pool_name: ZpoolName { id: ae2f89f7-35c8-4c05-940c-cdee6d8f8a12 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 42b2209d-f16a-4fca-a84d-54ecc9f13877 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.693996351Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.694045043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f, dataset_name: DatasetName { pool_name: ZpoolName { id: 42b2209d-f16a-4fca-a84d-54ecc9f13877 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b794d8c1-39ec-4743-9237-aeb9d36e8e47, dataset_name: DatasetName { pool_name: ZpoolName { id: 95ce52af-b115-45d4-acd1-3598b0bafa5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.762826688Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.762883809Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b794d8c1-39ec-4743-9237-aeb9d36e8e47, dataset_name: DatasetName { pool_name: ZpoolName { id: 95ce52af-b115-45d4-acd1-3598b0bafa5d (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ae414c2-10bd-42e6-9eda-6b0420abb343, dataset_name: DatasetName { pool_name: ZpoolName { id: ad348a98-27df-494e-af4b-560a8a421d64 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.848589028Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.84864104Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0ae414c2-10bd-42e6-9eda-6b0420abb343, dataset_name: DatasetName { pool_name: ZpoolName { id: ad348a98-27df-494e-af4b-560a8a421d64 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.943949905Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.944019567Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.944468929Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:15.944646743Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.072438293Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.07309318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.088486905Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.08869669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.088742641Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.113066776Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.113672811Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.113715012Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.127433204Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.127934517Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.127961328Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.270611408Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.345617153Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.345678275Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.345686105Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.345706486Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.345710495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.345714046Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.345717656Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.345910271Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.35135882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.356929663Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.361880101Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.366712394Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.371731273Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.376623769Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.524585746Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.52942091Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.533694879Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.538834082Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.542983958Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.548054258Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.552490062Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.558101726Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.563439023Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.568494453Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.574408534Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.580580313Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.586888675Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.592976571Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.600527425Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.606568Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.611790964Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.616624258Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.622032337Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.627350613Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.632374482Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.714680723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.82132501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.88094217Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.936733791Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.992241076Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.997258685Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:16.997276515Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.086742891Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 218, 154]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.086792302Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.092620882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.092650482Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.197203865Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 238, 230]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.197245097Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.200914471Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.200938352Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.317964975Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 145, 29]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.318009236Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.321764612Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.321790383Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.477844877Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 209, 89]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.477899539Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.480424333Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.480438074Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.581116337Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 214, 74]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.581168079Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.585628443Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.585646744Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.586624089Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.586642059Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.588043075Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.588057426Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.591473993Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.59486543Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.642039251Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.645865249Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.646035243Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.646052114Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.646195448Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.650791196Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.650828066Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.650836597Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.697123224Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.734215706Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.737052989Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.737073659Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.737219463Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.745661259Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:17.796626878Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:18.055953682Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:18.104165119Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:18.112124774Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:18.148717653Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:18.18366719Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:18.221750967Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:18.23083545Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService13\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::11\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::11\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:21.314183533Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:21.315099076Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService18\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::10\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::10\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:21.663778753Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:21.667602351Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService11\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::f\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::f\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:21.691179156Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:21.692027708Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService10\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_ad348a98-27df-494e-af4b-560a8a421d64/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1a\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"571ea486-e9ba-40d0-9076-006ea67325a7\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:21.76577073Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:21.890727157Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService12\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::19\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_95ce52af-b115-45d4-acd1-3598b0bafa5d/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::19\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"1fe30370-8df1-42aa-980a-99aefba5c3ea\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:22.068731625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService15\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::17\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_ae2f89f7-35c8-4c05-940c-cdee6d8f8a12/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::17\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"32c57d62-aa34-4b43-8e8e-9b0351f2008f\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:22.084225323Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService23\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::14\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_39005e7e-189e-49db-8463-a206621e7a54/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::14\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"1f55d83e-e023-463c-85d8-56a259c8064d\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:22.449897727Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:22.513888698Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:25.41208797Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService19\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::13\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_9d13b91d-5969-417a-bfea-ad89852618b9/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::13\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"14178711-08b8-4e8c-9a91-8b8dfcb1126b\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:25.490201665Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService20\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::15\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_a7771067-edf5-47c9-a972-69320dc9ee62/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::15\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"08b564d9-b71c-44b0-9080-2a55a913bfc2\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:25.565643231Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService22\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::16\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_096e5012-bddf-482f-8ad5-65f3b0930676/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::16\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"9afafd8b-33e3-4092-a92c-b6c9ec8c1808\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:25.584898125Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:25.585009388Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:25.713772661Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:25.919845139Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:26.241296978Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 38.254612ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:26.544278214Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 40.605455ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:26.797938132Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService14\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::18\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_42b2209d-f16a-4fca-a84d-54ecc9f13877/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::18\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"6c12baa5-4f49-4ce4-8629-0925e95d22f1\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:26.79824618Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 33.786321ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:26.963372358Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 61.565973ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:26.977764267Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 42.596783ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:26.987959979Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 39.071537ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:26.998867639Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 57.127074ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.011710258Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 48.639472ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.164223382Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 29.883667ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.177933534Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 113.605384ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.190682411Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 25.360959ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.20309972Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 125.401483ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.214544513Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:27.214620715Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 134.863513ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.227554997Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 144.562753ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.238484378Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 117.295741ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.249056369Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 104.508818ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.264226698Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService21\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::12\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_c919f4ac-100e-4c1e-bd24-76220f4848fc/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::12\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"848b45c0-8d60-4add-98c4-847c579861a2\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:27.266606069Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 96.816117ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.387929383Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 135.117651ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.399244183Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 84.843819ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.411001575Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 264.358612ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.42368056Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 99.038765ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.433955524Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 158.311777ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.445640234Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 143.660213ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.455036125Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 272.552749ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.46769462Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 101.940072ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.476555827Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 278.745188ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.48758472Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:27.487679663Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 281.531383ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.509544504Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 271.24056ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.521323656Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 297.996417ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.548947495Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 116.042183ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.564916665Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 557.281772ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.591538478Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 240.536756ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.610101364Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 583.916849ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.624495374Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 35.596236ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.634053489Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 70.501854ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.683748803Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 451.463663ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.697176838Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 519.60125ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.709524855Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 46.74729ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.718958507Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 323.701684ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.752887968Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 232.787359ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.765162763Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 112.505624ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.77752737Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 519.024485ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.856399294Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 569.117413ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.867068998Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 204.588112ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.876791737Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 635.835267ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.8874171Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 245.876007ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.898161745Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 284.972056ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:27.907258459Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 535.450011ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.014665155Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 991.010289ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.08966879Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.02324291s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.104761537Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 856.496148ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.160899588Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 1.056403381s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.172391073Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 886.537325ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.185879089Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 512.067312ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.204762413Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.079767034s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.220050996Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 613.261093ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.241432185Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 709.319875ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.392498611Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 606.153107ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.448814826Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.378414671s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.536489506Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 803.253927ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.568799235Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 499.452734ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.732468535Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 643.292748ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:28.891016183Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 797.364424ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.043366863Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.277081493s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.079855129Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 897.174188ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.095232464Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 1.266293569s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.110698011Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.298591889s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.125026799Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.489514096s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.14341436Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 549.305986ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.243881758Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 875.779477ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.257723254Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService16\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::d\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value=\"ec8d4748-9593-47c2-9fb9-59f95eb0cf38\"/>\n        <propval type=\"astring\" name=\"address\" value=\"[fd00:1122:3344:101::d]:12223\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:29.25834719Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:29.259202582Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.31265498s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.315752263Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.145225054s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.385827911Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 873.426289ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.54827967Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 51.932949ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.619419235Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 79.489205ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.691848984Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 206.026195ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.816536353Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 1.458166085s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.831905458Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 921.550429ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.871430772Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.436605548s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:29.94070368Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 680.142793ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.005698268Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 562.910467ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.034442695Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.486538383s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.148072961Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.14607985s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.371837703Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 821.538203ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.390258666Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 978.816318ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.436070932Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 639.018366ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.449354913Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.289693279s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.546099626Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 636.193082ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.611105994Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.310037449s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.644323866Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 681.061294ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.689197077Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 701.819799ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.70839549Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 675.358706ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:30.806765604Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:30.889339123Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58031"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:30.91299688Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"0fd4af31-0d41-4e11-b2f8-510d5461763c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58031","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":10697,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.404854943s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.102900793Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 510.867597ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.2266913Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 648.918577ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.259481972Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 600.527282ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.304341863Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.398263056s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.384531441Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.057721923s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.395985844Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.289812465s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.422783302Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 711.101757ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.435301224Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 1.244776304s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.495091608Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 995.121957ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.531471251Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.480873993s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.649307445Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 988.085614ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.750871291Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.331714989s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.850189579Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 667.87703ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.920028482Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.27412751s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.933024365Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.031229818s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:31.967489949Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.128182057s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.159426375Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.331669096s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.468556638Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 541.176869ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.520976943Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 716.170672ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.538909583Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 971.09432ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.599896878Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 978.744348ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.724575007Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 786.892368ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.75118655Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 1.318294186s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.76363753Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.447047845s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:32.794152843Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 593.559789ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.013572213Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 945.57244ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.077284668Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.161628548s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.14515335Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.117845157s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.195082491Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 591.812117ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.223043589Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.13653135s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.267708315Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.473432137s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.300996159Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 1.112271773s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.552849072Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 819.610544ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.583387286Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.076647734s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.616127696Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 642.161506ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.717526597Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.212528932s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.812798822Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.401511424s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:33.827206622Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 850.765183ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.038410562Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 1.164320013s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.096347069Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 524.134941ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.252326372Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.06600067s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.317989146Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.329769162s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.327253624Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.149819961s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.370182746Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 1.133444815s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.41319361Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 631.772087ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.421477012Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService17\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::e\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::e\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"8123\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:34.57025645Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:34.594352298Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 605.294391ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.675542601Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 767.823147ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.702313518Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.201635512s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.783462761Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.005712096s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.790399839Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 37.527257ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.865746872Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.27523292s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.897753764Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 134.916086ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:34.9108729Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 659.635943ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.034573195Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 297.457203ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.054929557Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.303696874s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.061590798Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.041167085s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.238342984Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 622.392226ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.270401947Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 837.894273ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.290118123Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 217.699179ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.361671219Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 695.422254ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.392028388Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 881.791774ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.478981539Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.226761464s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.529208798Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 901.606903ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.554874207Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 627.24044ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.587107784Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.394228664s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.667831755Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.112590499s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.70272342Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 896.73146ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.804644216Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 1.492217316s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.901805779Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.114093065s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:35.994724964Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 726.585647ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.097675005Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 1.216298859s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.158727492Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 904.447438ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.182642266Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 1.080654734s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.222649933Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.351099838s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.28841381Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 768.160442ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.370897577Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 834.325341ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.378318957Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 1.035741914s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.46493851Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.054165675s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.716976997Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService28\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::8\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::8]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.5:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:36.717454279Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService27\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::9\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::9]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.6:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:36.718164058Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:36.718226079Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:36.724991453Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 731.337171ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.769456904Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.059744145s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.832973704Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.38364054s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.851446428Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 32.081627ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:36.979640928Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 35.566944ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.015047236Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 78.406173ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.023665917Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 69.667401ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.07324039Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 681.218494ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.106822211Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 107.866654ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.114929379Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.27645178s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.155476Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 573.415059ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.164135042Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 686.349444ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.173817281Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 159.045629ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.183434888Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.263921853s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.248090887Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 343.404004ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.259329365Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService26\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::c\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:37.259644973Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService24\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:37.259680254Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService25\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:37.25989497Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:37.25990773Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:37.264541189Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:37.269331622Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 1.280967274s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.314437309Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 539.949484ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.35264045Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 1.089431774s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.385234446Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 940.141031ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.405194508Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 38.771929ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.418586602Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 70.032401ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.426830374Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 70.450402ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.443232815Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 75.571555ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.468835992Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 124.603541ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.51436118Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 781.548523ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.538571821Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.278472933s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.572698437Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 241.35863ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.584430438Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 105.77079ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.596118538Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 905.447104ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.617369743Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 272.314751ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.65153278Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.481243961s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.661150487Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 266.185986ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.714274569Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 651.403206ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.747542383Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 971.176019ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.781079133Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 871.879023ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.79809003Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 412.678445ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.837020419Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.279238166s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.871127804Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 1.094141758s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.902031808Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.295595828s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.913589954Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 528.796084ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.932453068Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 431.086574ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:37.99020263Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 549.319533ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.246125647Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 661.796547ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.260133897Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 768.372583ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.330622296Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 723.348347ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.354487858Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 886.2387ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.409146101Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 937.698153ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.43287117Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.012298327s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.44571774Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 1.084317495s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.47261059Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 1.069402468s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.485033378Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.448411192s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.522608173Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 1.411241478s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.533243926Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 1.321455092s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.608885007Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.487485143s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.683042689Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.3634078s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.764576381Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.14015948s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.805567503Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.342026231s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.863434418Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 623.949009ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:38.934208544Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 1.144366312s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.008654765Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 558.538764ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.090803413Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 1.310084705s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.1105762Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.023945633s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.152909796Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.393954374s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.162085292Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 839.282853ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.221158418Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.304268409s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.306531069Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 536.214713ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.380449726Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 740.922566ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.470334572Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 680.860281ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.569735203Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 1.138163843s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.58637534Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 604.202495ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.600597975Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 1.077101774s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.662303088Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 1.100570906s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.932376849Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 966.629704ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.945832694Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 598.840695ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.957236807Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 726.507682ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.968531676Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 582.961863ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:39.983739836Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 585.490853ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.072827183Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.221008006s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.1436403Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 1.111579664s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.171248858Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 541.096723ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.188266495Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.221866379s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.203433424Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 652.727739ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.221041936Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 557.300274ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.236806161Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 906.215002ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.253740315Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 968.678ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.272584809Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 1.093334281s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.43822153Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 819.579027ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.573176313Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.194308545s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.591889653Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 759.337176ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.608703914Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 930.334413ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.631496809Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 558.171105ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.681524322Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 891.535675ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.719560029Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 868.518861ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.742661182Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.37693185s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.760586671Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 625.361526ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.782223767Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 689.171968ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.813673614Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 895.717686ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.888285568Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 722.607928ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:40.928741347Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 649.150046ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.046132259Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.023959988s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.18098324Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 621.917407ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.260565032Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 1.116307719s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.281727695Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 806.909084ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.303848613Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 754.679698ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.379848453Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 740.432645ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.391883922Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 1.166132589s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.404206928Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 1.137631939s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.41984974Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 858.880173ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.435729897Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.177521064s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.515259788Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 901.479734ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.545287799Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.119035984s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.572203899Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 1.276121102s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.625249011Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.206292915s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.653245099Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 1.293574828s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.665969055Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 675.098939ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.705308214Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 862.757813ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.798245569Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.147253641s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.811459568Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.444733463s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:41.892852667Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 1.281111595s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.121661889Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 611.937273ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.141622991Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.149462898s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.151814932Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.413854463s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.161496431Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.022553863s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.215769693Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.251227094s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.306653046Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 1.155583409s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.390656951Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 519.373461ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.408876439Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 667.16228ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.458510293Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 1.472350308s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.568181477Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 763.70895ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.580460742Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 853.560741ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.672310639Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.125709332s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.701904088Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.422787142s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.711173286Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 885.01484ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.762673398Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.097643526s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.869703494Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 1.160810768s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.913269262Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 1.209626938s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.94123006Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 999.472917ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.96890577Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 760.4016ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:42.975802737Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 1.40482618s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.136548232Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.202400702s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.250604079Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 539.104462ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.315263208Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 831.264007ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.354050654Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 1.365442933s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.371088031Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 914.191638ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.417308787Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 674.200698ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.538514197Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 1.312587682s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.558164212Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 877.168866ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.571159005Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 613.763258ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.587110614Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.009741096s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.660830206Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 675.71386ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.748509025Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.253671861s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.840484896Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.217903012s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.871713787Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 996.505877ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.982169061Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 726.536042ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:43.993677087Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 1.255891638s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.056055397Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 1.288330952s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.08810833Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.115121487s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.148693345Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 757.127427ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.162220122Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 765.792733ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.197863547Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 1.486806704s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.215396726Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 693.999683ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.228823511Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 566.618676ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.343068253Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 799.101475ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.440983066Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 734.052964ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.467058865Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 934.071694ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.481776062Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 813.14007ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.558363608Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.444220142s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.684809332Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.33287735s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.731925811Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 703.54994ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.752282203Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 1.100710899s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.884586638Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 1.417605771s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.923028265Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 559.981683ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.935041463Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 1.463153257s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.945912852Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 568.63569ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.976513737Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 635.231464ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:44.99103546Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 666.262402ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.103146587Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.302355292s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.114515229Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.474315948s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.214640768Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 1.288089358s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.256850391Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.045851905s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.276225399Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473\") failed: Property not found. retry in 885.136337ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.324439716Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 724.508978ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.383995954Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 1.301048826s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.393441076Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 762.495969ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.426372162Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 1.005227834s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.467264071Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 650.540077ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.506518818Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 719.127656ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.557123847Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.010060761s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.63672766Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 892.361103ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.712850523Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 954.911678ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.781589127Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 1.451965728s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:45.999127649Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 953.699737ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.075561021Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 724.604916ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.118257286Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.394128099s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.139322207Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 825.891362ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.167552701Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 515.067538ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.199566073Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 572.855273ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.320532917Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 555.580871ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.33272504Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 677.655803ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.349755697Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 1.342021087s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.41844556Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 668.068528ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.426938138Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 623.334043ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.481307583Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 1.331818581s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.555157988Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 976.868571ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.614779978Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.154149745s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.656469338Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.02865868s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.704371306Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 1.227649324s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.71425359Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.364262716s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.724435621Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 925.788239ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.758375042Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 585.592961ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.85256785Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.006361424s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.897992975Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.046431442s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:46.905288932Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 1.124846818s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.00183893Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 1.289847421s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.036772026Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 1.293336835s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.04391401Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 870.63417ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.103983341Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 748.100038ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.114089781Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:47.28592829Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 720.516334ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.447814295Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 604.27446ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.461053134Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.267913263s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.546769124Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 581.827091ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.602486064Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.095252303s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.693779146Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38\") failed: Property not found. retry in 1.276083161s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.742370863Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9\") failed: Property not found. retry in 1.400847913s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.769892099Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa\") failed: Property not found. retry in 1.376486405s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.821476983Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 687.057279ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.871051035Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 1.428974607s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.894817315Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f\") failed: Property not found. retry in 1.43488647s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.911528333Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3\") failed: Property not found. retry in 1.234721739s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.954025464Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5\") failed: Property not found. retry in 1.077435498s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.963559019Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a\") failed: Property not found. retry in 1.447837518s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:47.982862684Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade\") failed: Property not found. retry in 1.334897057s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.074079895Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747\") failed: Property not found. retry in 1.196249252s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.098778039Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a\") failed: Property not found. retry in 767.50173ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.136271051Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5\") failed: Property not found. retry in 1.155015921s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.176419201Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f\") failed: Property not found. retry in 721.470966ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.192083933Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506\") failed: Property not found. retry in 706.333354ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.337271139Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da\") failed: Property not found. retry in 909.655822ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.347680196Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b\") failed: Property not found. retry in 767.375398ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.568506673Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47\") failed: Property not found. retry in 1.039447499s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.799340156Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343\") failed: Property not found. retry in 1.383735442s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:48.824263225Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee\") failed: Property not found. retry in 1.131732892s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:49.338470061Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528618682Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_0ae414c2-10bd-42e6-9eda-6b0420abb343"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528689454Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_bd856afa-64af-4f62-8083-7cec6f530e9b"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528700934Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_3e918472-0cb7-49b5-a622-8308a9ca651a"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528714894Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_b794d8c1-39ec-4743-9237-aeb9d36e8e47"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528721714Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_10cdd4df-0856-4bcf-af87-e5757529b0c3"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528730325Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_43c965bc-d0c5-4d77-8dee-7a6da492a2da"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528736105Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_9a7bb4b4-4ca3-47e0-82e2-2935db1acd6f"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528741575Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_7d57f350-6d84-441e-aca9-427498419506"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528747725Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_ed0566ef-ce2f-4a35-94d5-9861855ca4b5"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528753265Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_ec8d4748-9593-47c2-9fb9-59f95eb0cf38"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528759275Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_1af27ceb-2d94-42b4-abb9-e7054fec8ade"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528782306Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_d7310b7c-67e1-4338-8414-09460b7646e9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528792296Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_712a9949-23a6-4a4a-97da-2917d03922ee"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528798896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_b7dc045d-c7c8-45e1-af8a-fbd0f5f7890a"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528805117Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_1f204562-0cc1-49bf-9bd5-796c36691eb5"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528813087Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_87eb850e-70d3-4390-9948-908831c0c747"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.528818497Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_beb02584-2e2a-4158-9aa0-87876e6264fa"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.529585307Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_16207ecc-3887-4690-8f5f-8dfc1add1473"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.529598697Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_b8889290-9edc-47af-9bea-f3fc3227987f"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.53167238Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.544181411Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.568197338Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"4432019e-a6ee-4357-86e5-bdeb1b7cc45d","remote_addr":"[fd00:1122:3344:101::1]:55458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":35904176,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.586360424Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1236"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.586519638Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.586758324Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.592785778Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:680"}
{"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.59985189Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:693"}
{"msg":"rack_network_config: RackNetworkConfigV1 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.2.174,\n    infra_ip_last: 10.151.2.174,\n    ports: [\n        PortConfigV1 {\n            addresses: [\n                V4(\n                    Ipv4Network {\n                        addr: 10.151.2.174,\n                        prefix: 24,\n                    },\n                ),\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Network {\n                            addr: 0.0.0.0,\n                            prefix: 0,\n                        },\n                    ),\n                    nexthop: 10.151.2.100,\n                    vlan_id: None,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Network {\n        addr: fd00:1122:3344:100::,\n        prefix: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:50.68404862Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:817"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:53.62152341Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:53.769923607Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:53.945289318Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:54.125666606Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:54.263096563Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:54.398636391Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:54.7341192Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:54.966606416Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:55.152969898Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:55.513325946Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:56.171833974Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:56.987735391Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:57.851303341Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:58.763959981Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:44:59.362563732Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:44:59.37806535Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:44:59.608139954Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:45:00.614754815Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:45:00.803556159Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:00.948828577Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42681"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:00.949175406Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"05c420b5-abc5-4d13-a4ed-575c5159b0b1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42681","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":40,"response_code":"200"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:01.268407908Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:45:01.618506752Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/240cabb9-b684-4172-8606-d31cd2279cd6/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:02.549341989Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-14T09:45:02.622961818Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5f617488-2ef8-4cf1-81f7-2b249f5b34c0): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:45:02.804631169Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1623130b-4b07-46da-a08d-a1e6e21f1ae8\", \"content-length\": \"173\", \"date\": \"Tue, 14 May 2024 09:45:03 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"240cabb9-b684-4172-8606-d31cd2279cd6\\\"\", request_id: \"1623130b-4b07-46da-a08d-a1e6e21f1ae8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:03.817166502Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:06.933304816Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.166583863Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:42055"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.167295001Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.265690086Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"37322965-5213-4322-b1f7-b450c2d0ff13","remote_addr":"[fd00:1122:3344:101::c]:42055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":98884,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.26741761Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.267451941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.275274152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.275880727Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"939f2b38-806b-4c77-90d2-df1f6aa71920","remote_addr":"[fd00:1122:3344:101::c]:42055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9473,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.830042068Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54178"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.834271476Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b5cecdd2-5810-47a4-acef-30ed4d68f165, kind: Service { id: 7d57f350-6d84-441e-aca9-427498419506 }, name: Name(\"external-dns-7d57f350-6d84-441e-aca9-427498419506\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 209, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c3a0d6a8-690b-4d68-9c53-ea94903a313d, kind: Service { id: bd856afa-64af-4f62-8083-7cec6f530e9b }, name: Name(\"external-dns-bd856afa-64af-4f62-8083-7cec6f530e9b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e69104d-a1e3-43ae-9fa7-d0ba3efaba6d, kind: Service { id: d7310b7c-67e1-4338-8414-09460b7646e9 }, name: Name(\"nexus-d7310b7c-67e1-4338-8414-09460b7646e9\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: afa350c1-4cae-457d-b6fa-02b99bddeae1, kind: Service { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3 }, name: Name(\"nexus-10cdd4df-0856-4bcf-af87-e5757529b0c3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 29])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b34dcba8-9ce7-4e76-8638-6efaa0e9a52e, kind: Service { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade }, name: Name(\"nexus-1af27ceb-2d94-42b4-abb9-e7054fec8ade\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 74])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.834459001Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.834707967Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.834734278Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.834741858Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.834751999Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.834758679Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.834783229Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f755b494-f47c-42db-b204-24124d82a89e","remote_addr":"[fd00:1122:3344:101::c]:54178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":4577,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:07.837196141Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:889"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:08.04521934Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:62493"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:08.045325992Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:08.139112759Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"5bc69e61-9db6-4e61-893b-e3734958b41c","remote_addr":"[fd00:1122:3344:101::c]:62493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":93821,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:08.139462338Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/112d7827-193f-456c-94a5-19163d29475e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:08.13954Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:08.140061723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:08.140798582Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"8b001f7e-d75e-49d3-9fdd-4cb41660e0b7","remote_addr":"[fd00:1122:3344:101::c]:62493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1450,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711018306Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37099"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711280473Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b5cecdd2-5810-47a4-acef-30ed4d68f165, kind: Service { id: 7d57f350-6d84-441e-aca9-427498419506 }, name: Name(\"external-dns-7d57f350-6d84-441e-aca9-427498419506\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 209, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c3a0d6a8-690b-4d68-9c53-ea94903a313d, kind: Service { id: bd856afa-64af-4f62-8083-7cec6f530e9b }, name: Name(\"external-dns-bd856afa-64af-4f62-8083-7cec6f530e9b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e69104d-a1e3-43ae-9fa7-d0ba3efaba6d, kind: Service { id: d7310b7c-67e1-4338-8414-09460b7646e9 }, name: Name(\"nexus-d7310b7c-67e1-4338-8414-09460b7646e9\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: afa350c1-4cae-457d-b6fa-02b99bddeae1, kind: Service { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3 }, name: Name(\"nexus-10cdd4df-0856-4bcf-af87-e5757529b0c3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 29])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b34dcba8-9ce7-4e76-8638-6efaa0e9a52e, kind: Service { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade }, name: Name(\"nexus-1af27ceb-2d94-42b4-abb9-e7054fec8ade\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 74])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711333694Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711645252Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711658022Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711664612Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711676583Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711690293Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.711716714Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"21f2c083-7138-47b5-9c14-4ba24dd8bbf6","remote_addr":"[fd00:1122:3344:101::c]:37099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":628,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.723161067Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60138"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.734667622Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b5cecdd2-5810-47a4-acef-30ed4d68f165, kind: Service { id: 7d57f350-6d84-441e-aca9-427498419506 }, name: Name(\"external-dns-7d57f350-6d84-441e-aca9-427498419506\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 209, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c3a0d6a8-690b-4d68-9c53-ea94903a313d, kind: Service { id: bd856afa-64af-4f62-8083-7cec6f530e9b }, name: Name(\"external-dns-bd856afa-64af-4f62-8083-7cec6f530e9b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e69104d-a1e3-43ae-9fa7-d0ba3efaba6d, kind: Service { id: d7310b7c-67e1-4338-8414-09460b7646e9 }, name: Name(\"nexus-d7310b7c-67e1-4338-8414-09460b7646e9\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: afa350c1-4cae-457d-b6fa-02b99bddeae1, kind: Service { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3 }, name: Name(\"nexus-10cdd4df-0856-4bcf-af87-e5757529b0c3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 29])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b34dcba8-9ce7-4e76-8638-6efaa0e9a52e, kind: Service { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade }, name: Name(\"nexus-1af27ceb-2d94-42b4-abb9-e7054fec8ade\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 74])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.734701834Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.738168242Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.738193773Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.738208244Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.738221774Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.738233414Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.738248625Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"221c2289-c449-4391-accc-357f36b39cac","remote_addr":"[fd00:1122:3344:101::b]:60138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":14961,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.800854681Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56117"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.801136058Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b5cecdd2-5810-47a4-acef-30ed4d68f165, kind: Service { id: 7d57f350-6d84-441e-aca9-427498419506 }, name: Name(\"external-dns-7d57f350-6d84-441e-aca9-427498419506\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 209, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c3a0d6a8-690b-4d68-9c53-ea94903a313d, kind: Service { id: bd856afa-64af-4f62-8083-7cec6f530e9b }, name: Name(\"external-dns-bd856afa-64af-4f62-8083-7cec6f530e9b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e69104d-a1e3-43ae-9fa7-d0ba3efaba6d, kind: Service { id: d7310b7c-67e1-4338-8414-09460b7646e9 }, name: Name(\"nexus-d7310b7c-67e1-4338-8414-09460b7646e9\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: afa350c1-4cae-457d-b6fa-02b99bddeae1, kind: Service { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3 }, name: Name(\"nexus-10cdd4df-0856-4bcf-af87-e5757529b0c3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 29])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b34dcba8-9ce7-4e76-8638-6efaa0e9a52e, kind: Service { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade }, name: Name(\"nexus-1af27ceb-2d94-42b4-abb9-e7054fec8ade\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 74])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.801185229Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.801235251Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.801247201Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.801259291Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.801279972Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.801295612Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:09.801314873Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f3724341-ae62-4478-acb9-ea9be7c4e945","remote_addr":"[fd00:1122:3344:101::a]:56117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":388,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:30.987971481Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47203"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:30.988131246Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"89d62437-643e-4797-a6bf-bccd4b7b358b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47203","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":59,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.753722374Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59166"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.834173308Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"67bcaf5f-5d47-492b-9b25-63ceab0bbaf0","remote_addr":"[fd00:1122:3344:101::a]:59166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":80395,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.834385064Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36118"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.834415335Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60078"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.839015243Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a5db7146-0ddb-4f50-a7bb-b77483374119","remote_addr":"[fd00:1122:3344:101::a]:59166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":4430,"response_code":"200"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.860375981Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/112d7827-193f-456c-94a5-19163d29475e/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.860440623Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.861867039Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.86265803Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.862804713Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"bf6da70a-c81a-4a0b-af1e-cf5e6c314f14","remote_addr":"[fd00:1122:3344:101::a]:59166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":2826,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.891337146Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e2934d8c-c75c-46be-bc8e-09e42a47c661","remote_addr":"[fd00:1122:3344:101::b]:36118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":56837,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.895691167Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9eb4286a-a6ed-4685-8cc4-61cc837fc6ee","remote_addr":"[fd00:1122:3344:101::b]:36118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1059,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.942864758Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0342e2e4-4342-4413-a003-2bfa06cd2081","remote_addr":"[fd00:1122:3344:101::c]:60078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":108287,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:45:33.943277008Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"39e99e76-6cc4-4e44-a177-2201eef06566","remote_addr":"[fd00:1122:3344:101::c]:60078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:01.020185249Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44202"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:01.020290161Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ee01c67a-6838-46dc-825a-0a747288bcd9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44202","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":82,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:03.821938626Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44323"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:03.875518761Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ae6af296-291f-40b3-9ab3-524052e07287","remote_addr":"[fd00:1122:3344:101::c]:44323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":53512,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:03.875833458Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6d961d2b-516c-4507-9984-847cdbd7e7d7","remote_addr":"[fd00:1122:3344:101::c]:44323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:03.927749931Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47867"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:03.979251343Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9c1ba0f5-8b67-44cf-b403-721d2f90d2d9","remote_addr":"[fd00:1122:3344:101::a]:47867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":51381,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:03.980434863Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"919bba37-4485-4bd7-a086-5a4006944b56","remote_addr":"[fd00:1122:3344:101::a]:47867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:04.010456093Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:43058"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:04.060268711Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"dc2bed7a-44bb-43a9-a7ac-ea9d953f6bab","remote_addr":"[fd00:1122:3344:101::b]:43058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49715,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:04.06059076Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c86c87a7-7bfc-441f-8f15-69ddc6787958","remote_addr":"[fd00:1122:3344:101::b]:43058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":116,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:31.051127871Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58546"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:31.051255754Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"5000662e-23b6-4426-9232-ae243d1a32a7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58546","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.622945027Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64085"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.671303098Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6bec7ae2-d6f8-4303-b8f2-5b99b1789764","remote_addr":"[fd00:1122:3344:101::b]:64085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48305,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.671484832Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48017"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.675224539Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b6fd541a-fa4c-4976-b4bb-5f45b9ab8e9a","remote_addr":"[fd00:1122:3344:101::b]:64085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3594,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.717712709Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5d0e5246-d9e8-406a-bfdc-7f4805b270f1","remote_addr":"[fd00:1122:3344:101::a]:48017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46148,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.718082778Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"937505bc-1d58-4ecf-95ab-29b7ba2864f9","remote_addr":"[fd00:1122:3344:101::a]:48017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":165,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.735325471Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60760"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.779842663Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5d49a082-8940-4709-a52a-5ec0ebe75668","remote_addr":"[fd00:1122:3344:101::c]:60760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":44458,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:33.780180862Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f8bf819b-e316-416a-bc03-235460d2a022","remote_addr":"[fd00:1122:3344:101::c]:60760","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":144,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:50.903737273Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:45846"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:46:50.916234404Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"82a7380e-d926-4e0f-a70c-d844b3b1f745","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":12432,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:01.081876476Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40398"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:01.081968909Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b7067b5c-d838-4c4d-83fc-577225b3a53e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40398","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":85,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:03.855177883Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43079"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:03.907994078Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"fa778799-330a-417c-8efa-1edb7e70882d","remote_addr":"[fd00:1122:3344:101::c]:43079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":52734,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:03.908154762Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44993"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:03.95873145Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"15fbcea3-9d18-4816-b925-e34c75b22ec7","remote_addr":"[fd00:1122:3344:101::b]:44993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":50491,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:03.959086459Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"25b52e41-44b2-4084-a760-25c85803fcb5","remote_addr":"[fd00:1122:3344:101::b]:44993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:03.959150411Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0ab126cf-8117-491b-9e30-e9e3d3c268dd","remote_addr":"[fd00:1122:3344:101::c]:43079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":213,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:04.138628797Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44751"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:04.18316073Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"71395211-6beb-4b68-b67e-b7f040d57924","remote_addr":"[fd00:1122:3344:101::a]:44751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":44440,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:04.184212697Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1847ae36-e671-4f66-9e15-e8cab537a907","remote_addr":"[fd00:1122:3344:101::a]:44751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":877,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:20.903257627Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"f5d203b2-9465-4532-b62e-745b8fc8f4f4","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":684,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:31.115123056Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33289"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:31.115213948Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"9a95376b-ec4f-4b47-8f64-e69fbd2b1dc3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33289","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":86,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.635604755Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40004"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.685247658Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6d358129-19f6-4eab-a999-37d5e3f69313","remote_addr":"[fd00:1122:3344:101::b]:40004","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49575,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.685398152Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54965"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.688867941Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"eb1da26a-075d-4960-94ee-108b90cc07a6","remote_addr":"[fd00:1122:3344:101::b]:40004","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3264,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.731565657Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"7b3919f5-5e61-40e3-9ac7-57699da534a9","remote_addr":"[fd00:1122:3344:101::c]:54965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46056,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.732642514Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7498ac9d-b278-4f49-b9ca-6bc924eccf38","remote_addr":"[fd00:1122:3344:101::c]:54965","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":853,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.73832427Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39734"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.783562141Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c8ceaa92-ea98-4aca-ad7e-f0648ea41f22","remote_addr":"[fd00:1122:3344:101::a]:39734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":45150,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:33.783858108Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e58824f7-8d67-4aeb-b8d3-7519ea5b1a97","remote_addr":"[fd00:1122:3344:101::a]:39734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:47:50.90356738Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"58e4a3aa-c30b-4821-b774-5c4b00d2f24e","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":533,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:01.148529678Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65240"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:01.148627891Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"d5df1c95-1cba-4b73-a577-780c24afdef0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65240","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":65,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.75447228Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61755"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.805789006Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a780facc-c34d-4dd4-8910-6f2dd093436a","remote_addr":"[fd00:1122:3344:101::c]:61755","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":51252,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.806029012Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"46c3bcfc-e1bf-4467-8f3a-c940f8813691","remote_addr":"[fd00:1122:3344:101::c]:61755","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.844001037Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:57322"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.895496928Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c86ce060-9883-4ba6-826a-079a119df351","remote_addr":"[fd00:1122:3344:101::b]:57322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":51383,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.895803146Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"fadb1009-bdf8-4841-a940-99d4caafbfc0","remote_addr":"[fd00:1122:3344:101::b]:57322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.910690748Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35703"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.957538791Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0213566c-cf81-4b93-866f-150bcbb6a313","remote_addr":"[fd00:1122:3344:101::a]:35703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46762,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:03.957790637Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"380b1e9f-8829-49d1-93da-3b712664f74c","remote_addr":"[fd00:1122:3344:101::a]:35703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:20.903607237Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"366fcda4-fc11-4043-9696-4cc5f4da30a7","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1051,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:31.18035168Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52808"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:31.180477013Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"0bfb93fd-33c2-46fb-9214-58c660028b16","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52808","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.634861916Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:43099"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.686402618Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a90c3b94-66bb-41f8-9ca7-5f5d57bd7ae2","remote_addr":"[fd00:1122:3344:101::b]:43099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":51478,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.686532492Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35225"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.690805641Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"30a165d3-228c-4dd8-a372-55de376fa4c2","remote_addr":"[fd00:1122:3344:101::b]:43099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":4078,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.732646165Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"33ade52b-f00c-405c-bcd6-5051f2e21b9f","remote_addr":"[fd00:1122:3344:101::c]:35225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46027,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.732994934Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"625409ce-3cd1-46e9-9957-2c112788c275","remote_addr":"[fd00:1122:3344:101::c]:35225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":130,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.802460977Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53321"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.847167504Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"610bf0e9-13cb-4bd0-8f8e-398e4f5b42df","remote_addr":"[fd00:1122:3344:101::a]:53321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":44642,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:33.847389Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9da99f29-777d-4d28-ad10-abcfaea8cb78","remote_addr":"[fd00:1122:3344:101::a]:53321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:48:50.908308393Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"4304391d-7264-4be5-87d6-adc3ad942f04","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1361,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:01.211044853Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51006"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:01.211165477Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"47138253-8e82-4960-a437-ec669e5e3de4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51006","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:03.826866198Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56800"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:03.82694812Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48420"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:03.872983242Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c8c45af8-355c-4b19-b5bf-4fbca8537cbc","remote_addr":"[fd00:1122:3344:101::c]:56800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46016,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:03.876250596Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7466514e-f792-470c-81c8-f8c378f4fdc7","remote_addr":"[fd00:1122:3344:101::c]:56800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":440,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:03.920743197Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4ff948d3-957f-41f6-b097-90831ac620f7","remote_addr":"[fd00:1122:3344:101::b]:48420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":93660,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:03.921122497Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b3d91cfa-c73a-40ae-a518-c3838e4e6271","remote_addr":"[fd00:1122:3344:101::b]:48420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":142,"response_code":"200"}
{"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:04.065324677Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:04.066503158Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:04.10671055Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999646s","next_cleanup":"Instant { tv_sec: 1953, tv_nsec: 481619869 }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:04.185017299Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52553"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:04.233883283Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"4ee94650-b6da-45e5-bc58-847330e9f705","remote_addr":"[fd00:1122:3344:101::a]:52553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48770,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:04.234208361Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e6f36dd7-5293-4bdd-8c76-af34ac909500","remote_addr":"[fd00:1122:3344:101::a]:52553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":118,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:20.945048141Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"08b643e8-7213-4088-b31d-a3b79e07cb21","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1288,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:31.244437866Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50292"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:31.244557629Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"31691f0d-1698-4241-aff1-e553da38cfbd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50292","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.725498073Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59729"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.775388903Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ac8ea729-4bdb-4869-9086-4ddd114f6656","remote_addr":"[fd00:1122:3344:101::c]:59729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49820,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.775735122Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"2eab15b1-5ba5-4e97-aad1-492dce8880de","remote_addr":"[fd00:1122:3344:101::c]:59729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":137,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.77604398Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58361"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.820439519Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0e8e3050-a489-4f43-912c-3866e0faf33e","remote_addr":"[fd00:1122:3344:101::a]:58361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":44330,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.820743897Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"670b8771-bfa5-4b2d-85c4-0d8f962f5e58","remote_addr":"[fd00:1122:3344:101::a]:58361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":124,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.838912693Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62340"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.883149078Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"2888f17a-9b36-4084-9454-3dc71a3ca5f3","remote_addr":"[fd00:1122:3344:101::b]:62340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":44165,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:33.883408765Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a92be0c8-3e77-45ab-984e-6b68d44140a6","remote_addr":"[fd00:1122:3344:101::b]:62340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:47.285982472Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:49:50.944863363Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"ebdbce55-1c2d-4f34-aa79-7ce3d523cb76","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":703,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:01.276230549Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41002"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:01.276385243Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"845cfead-8a99-4e0c-8424-d96ac786e375","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41002","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":121,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.847639124Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39926"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896247741Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"2c874986-51c1-429f-8b0a-bf94aa73dd83","remote_addr":"[fd00:1122:3344:101::c]:39926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48544,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896362034Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:42055"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896655982Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b5cecdd2-5810-47a4-acef-30ed4d68f165, kind: Service { id: 7d57f350-6d84-441e-aca9-427498419506 }, name: Name(\"external-dns-7d57f350-6d84-441e-aca9-427498419506\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 209, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c3a0d6a8-690b-4d68-9c53-ea94903a313d, kind: Service { id: bd856afa-64af-4f62-8083-7cec6f530e9b }, name: Name(\"external-dns-bd856afa-64af-4f62-8083-7cec6f530e9b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e69104d-a1e3-43ae-9fa7-d0ba3efaba6d, kind: Service { id: d7310b7c-67e1-4338-8414-09460b7646e9 }, name: Name(\"nexus-d7310b7c-67e1-4338-8414-09460b7646e9\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: afa350c1-4cae-457d-b6fa-02b99bddeae1, kind: Service { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3 }, name: Name(\"nexus-10cdd4df-0856-4bcf-af87-e5757529b0c3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 29])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b34dcba8-9ce7-4e76-8638-6efaa0e9a52e, kind: Service { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade }, name: Name(\"nexus-1af27ceb-2d94-42b4-abb9-e7054fec8ade\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 74])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896683342Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f7d7fd7e-f68b-4ef4-80ca-db2e397d1628","remote_addr":"[fd00:1122:3344:101::c]:39926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":190,"response_code":"200"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896711713Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896751664Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896769665Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896787375Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896799405Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896807206Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:03.896819666Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c1789bd2-e8f4-45e8-b779-190e42a5076e","remote_addr":"[fd00:1122:3344:101::c]:42055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":386,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.017664567Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52937"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.017911203Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b5cecdd2-5810-47a4-acef-30ed4d68f165, kind: Service { id: 7d57f350-6d84-441e-aca9-427498419506 }, name: Name(\"external-dns-7d57f350-6d84-441e-aca9-427498419506\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 209, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c3a0d6a8-690b-4d68-9c53-ea94903a313d, kind: Service { id: bd856afa-64af-4f62-8083-7cec6f530e9b }, name: Name(\"external-dns-bd856afa-64af-4f62-8083-7cec6f530e9b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e69104d-a1e3-43ae-9fa7-d0ba3efaba6d, kind: Service { id: d7310b7c-67e1-4338-8414-09460b7646e9 }, name: Name(\"nexus-d7310b7c-67e1-4338-8414-09460b7646e9\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: afa350c1-4cae-457d-b6fa-02b99bddeae1, kind: Service { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3 }, name: Name(\"nexus-10cdd4df-0856-4bcf-af87-e5757529b0c3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 29])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b34dcba8-9ce7-4e76-8638-6efaa0e9a52e, kind: Service { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade }, name: Name(\"nexus-1af27ceb-2d94-42b4-abb9-e7054fec8ade\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 74])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.017967285Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.018014226Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.018030816Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.018052877Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.018059947Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.018070697Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.018087218Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e350ca37-7cc2-4351-81a3-fc110c95339b","remote_addr":"[fd00:1122:3344:101::b]:52937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":392,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.05596435Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64534"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.106833475Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"56ac1dd1-5b20-41b9-9a5b-f6ab7cdbce4e","remote_addr":"[fd00:1122:3344:101::b]:64534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":50826,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.106977779Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49191"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.107047481Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43360"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.107362769Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b5cecdd2-5810-47a4-acef-30ed4d68f165, kind: Service { id: 7d57f350-6d84-441e-aca9-427498419506 }, name: Name(\"external-dns-7d57f350-6d84-441e-aca9-427498419506\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 209, 89])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c3a0d6a8-690b-4d68-9c53-ea94903a313d, kind: Service { id: bd856afa-64af-4f62-8083-7cec6f530e9b }, name: Name(\"external-dns-bd856afa-64af-4f62-8083-7cec6f530e9b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 230])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0e69104d-a1e3-43ae-9fa7-d0ba3efaba6d, kind: Service { id: d7310b7c-67e1-4338-8414-09460b7646e9 }, name: Name(\"nexus-d7310b7c-67e1-4338-8414-09460b7646e9\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 218, 154])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: afa350c1-4cae-457d-b6fa-02b99bddeae1, kind: Service { id: 10cdd4df-0856-4bcf-af87-e5757529b0c3 }, name: Name(\"nexus-10cdd4df-0856-4bcf-af87-e5757529b0c3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 29])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b34dcba8-9ce7-4e76-8638-6efaa0e9a52e, kind: Service { id: 1af27ceb-2d94-42b4-abb9-e7054fec8ade }, name: Name(\"nexus-1af27ceb-2d94-42b4-abb9-e7054fec8ade\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 74])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.10740083Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"73658f07-83d4-4067-8eb2-92b8480f2c10","remote_addr":"[fd00:1122:3344:101::b]:64534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":235,"response_code":"200"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.10742483Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.107469602Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.111244719Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.111354011Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.111374012Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.111390422Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.111646389Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"19af518e-8437-4e53-a018-180cc10a6cf5","remote_addr":"[fd00:1122:3344:101::a]:49191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":4577,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.157137656Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"00c1352d-b349-44be-9902-6f134d62fa88","remote_addr":"[fd00:1122:3344:101::a]:43360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49960,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:04.157537356Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8e5c7462-867e-4762-ad05-73300da99fe4","remote_addr":"[fd00:1122:3344:101::a]:43360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":197,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:20.944343056Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"3a69b1d7-b18c-4e64-9f6f-cbe45ce0667d","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":570,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:31.30836021Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62772"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:31.308460183Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"8dfbd103-e9ea-47ce-bc6f-8693a7f9acee","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62772","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.643328409Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:38790"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.69210866Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"58d16d0e-3029-40ce-97d6-a4aac9a6166d","remote_addr":"[fd00:1122:3344:101::a]:38790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48700,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.692207212Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:41238"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.738699695Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"158e8328-7252-4902-b408-49cdf9fac4bd","remote_addr":"[fd00:1122:3344:101::b]:41238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46433,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.738843439Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55651"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.742458272Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7ac311fa-6a54-45a0-8a24-933b6164d448","remote_addr":"[fd00:1122:3344:101::a]:38790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3399,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.742556864Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a9783d7b-6572-48a1-ba50-f7e2cc7f01a2","remote_addr":"[fd00:1122:3344:101::b]:41238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3440,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.787029575Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5c541243-a862-41f3-ab38-7b7c82a02ee3","remote_addr":"[fd00:1122:3344:101::c]:55651","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48118,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:33.787401355Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"9a9ea983-df0a-45b2-9256-a7fc01d82629","remote_addr":"[fd00:1122:3344:101::c]:55651","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":160,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:50:50.94386053Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"bbabeb2a-7cb7-4567-bd58-02f488900c7c","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":553,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:01.339274139Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39753"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:01.339368922Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"9894e76e-8f67-4323-b5dc-c68be036c4b9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39753","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":63,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:03.836411738Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50161"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:03.890163568Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"932da1a2-0bc4-44c0-ac39-a154293b8f7a","remote_addr":"[fd00:1122:3344:101::c]:50161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":53689,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:03.891260746Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"135a3479-fa0b-4dfa-b2e5-481665706757","remote_addr":"[fd00:1122:3344:101::c]:50161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":900,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:03.90819366Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60390"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:03.957730101Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f3beffdf-9e11-4bd4-a3de-7aa2248c8006","remote_addr":"[fd00:1122:3344:101::b]:60390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49447,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:03.957847605Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53597"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:03.961504618Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"21d96105-a934-48bd-a80f-b63e9ad1956e","remote_addr":"[fd00:1122:3344:101::b]:60390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3476,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:04.004626695Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"95bc9226-02c7-44aa-93d2-1cd9cd1e74df","remote_addr":"[fd00:1122:3344:101::a]:53597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46678,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:04.004979754Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"67790081-02fe-46ff-a67c-e970fac5f939","remote_addr":"[fd00:1122:3344:101::a]:53597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":131,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.262310045Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:32960"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.278810558Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:546","metadata":"InstanceMetadata { silo_id: 82af3ca8-5091-4869-8146-a9112885b828, project_id: 4ed919d2-cc7b-4120-bf73-e8ea0ff719fe }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-14T09:51:09.828338Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(f26a90e0-2719-418f-ba7b-06c7da6036f0), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-14T09:51:09.852064397Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 49ed10a6-8516-40ff-a573-5f419cb03dfc, kind: Instance { id: 1f8794d5-895b-43ae-93ed-692f86bfccb7 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 247, 62, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(2422046), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 49ed10a6-8516-40ff-a573-5f419cb03dfc, kind: Instance { id: 1f8794d5-895b-43ae-93ed-692f86bfccb7 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 247, 62, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(2422046), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 49ed10a6-8516-40ff-a573-5f419cb03dfc, kind: Instance { id: 1f8794d5-895b-43ae-93ed-692f86bfccb7 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 247, 62, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(2422046), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(2422046))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 49ed10a6-8516-40ff-a573-5f419cb03dfc, kind: Instance { id: 1f8794d5-895b-43ae-93ed-692f86bfccb7 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 247, 62, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(2422046), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-70d3fca55a4\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 17aa7c57-ea22-4897-a188-6ddd36f0db8a, read_only_parent: Some(Volume { block_size: 512, id: ecd5b714-8efd-47a5-a08c-a385d8595ac3, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0e2134da-e682-4e76-a47d-4449a95a8edd, key: Some(\"YQ8Jzh6H6ZL7hVOL3Gy+lSQaGmePLhNYiDpTwThsZYw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::17]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 17aa7c57-ea22-4897-a188-6ddd36f0db8a, key: Some(\"EJCYRHrF7MxWReI3qFkfDucLYfuuwo1P4Vin0J5Qj1E=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAWZU6uWK5YAABlTq5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAFmVOrliuWAAAZU6uWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxZjg3OTRkNS04OTViLTQzYWUtOTNlZC02OTJmODZiZmNjYjciLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRGdOUXZveW5vK0FPUldneDdEekdiR2NSTG1uSVZlT3VzZ1g3a1lhRkozRCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.278919861Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:580","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.279959228Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 49ed10a6-8516-40ff-a573-5f419cb03dfc, kind: Instance { id: 1f8794d5-895b-43ae-93ed-692f86bfccb7 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 247, 62, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(2422046), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 49ed10a6-8516-40ff-a573-5f419cb03dfc, kind: Instance { id: 1f8794d5-895b-43ae-93ed-692f86bfccb7 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 247, 62, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(2422046), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 49ed10a6-8516-40ff-a573-5f419cb03dfc, kind: Instance { id: 1f8794d5-895b-43ae-93ed-692f86bfccb7 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 247, 62, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(2422046), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(2422046))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 49ed10a6-8516-40ff-a573-5f419cb03dfc, kind: Instance { id: 1f8794d5-895b-43ae-93ed-692f86bfccb7 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 247, 62, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(2422046), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-70d3fca55a4\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 17aa7c57-ea22-4897-a188-6ddd36f0db8a, read_only_parent: Some(Volume { block_size: 512, id: ecd5b714-8efd-47a5-a08c-a385d8595ac3, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0e2134da-e682-4e76-a47d-4449a95a8edd, key: Some(\"YQ8Jzh6H6ZL7hVOL3Gy+lSQaGmePLhNYiDpTwThsZYw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::17]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 17aa7c57-ea22-4897-a188-6ddd36f0db8a, key: Some(\"EJCYRHrF7MxWReI3qFkfDucLYfuuwo1P4Vin0J5Qj1E=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAWZU6uWK5YAABlTq5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAFmVOrliuWAAAZU6uWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxZjg3OTRkNS04OTViLTQzYWUtOTNlZC02OTJmODZiZmNjYjciLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRGdOUXZveW5vK0FPUldneDdEekdiR2NSTG1uSVZlT3VzZ1g3a1lhRkozRCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(f26a90e0-2719-418f-ba7b-06c7da6036f0), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-14T09:51:09.852064397Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-14T09:51:09.828338Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.280591094Z","hostname":"sock","pid":1590,"uri":"/instances/1f8794d5-895b-43ae-93ed-692f86bfccb7","method":"PUT","req_id":"91a65a15-a607-489f-8c04-98c11f6c1cd1","remote_addr":"[fd00:1122:3344:101::a]:32960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":18244,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.389306734Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61674"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.39149539Z","hostname":"sock","pid":1590,"uri":"/instances/1f8794d5-895b-43ae-93ed-692f86bfccb7/state","method":"PUT","req_id":"c8416eb4-790a-4953-9a22-3d519dd0e5ce","remote_addr":"[fd00:1122:3344:101::a]:61674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":2106,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.415220999Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 247, 62, 20]), slot: 0, vni: Vni { inner: 2422046 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.41528655Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.452164537Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:10.47449936Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlInstance0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1:0\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"12400\"/>\n        <propval type=\"astring\" name=\"metric_addr\" value=\"[fd00:1122:3344:101::b]:12221\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:11.853767623Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:11.854063721Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 67.293982ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:12.052644647Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 53.723103ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:12.136963721Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 200.346745ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:12.205978941Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 249.482112ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:12.417019867Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 883.725022ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:12.682239902Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 1.156923119s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:13.576219814Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 1.192502644s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:14.743733743Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 793.881562ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:15.94659597Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 1.392568325s","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:16.75210468Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.175666491Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 41.369963ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:18.184144888Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 128.483191ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:18.234952412Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\") failed: Property not found. retry in 167.812163ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:18.373351644Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.551884295Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.593814091Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"}
{"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAWZU6uWK5YAABlTq5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAFmVOrliuWAAAZU6uWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIxZjg3OTRkNS04OTViLTQzYWUtOTNlZC02OTJmODZiZmNjYjciLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRGdOUXZveW5vK0FPUldneDdEekdiR2NSTG1uSVZlT3VzZ1g3a1lhRkozRCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-70d3fca55a4\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 17aa7c57-ea22-4897-a188-6ddd36f0db8a, read_only_parent: Some(Volume { block_size: 512, id: ecd5b714-8efd-47a5-a08c-a385d8595ac3, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0e2134da-e682-4e76-a47d-4449a95a8edd, key: Some(\"YQ8Jzh6H6ZL7hVOL3Gy+lSQaGmePLhNYiDpTwThsZYw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::17]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 17aa7c57-ea22-4897-a188-6ddd36f0db8a, key: Some(\"EJCYRHrF7MxWReI3qFkfDucLYfuuwo1P4Vin0J5Qj1E=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19003\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 1f8794d5-895b-43ae-93ed-692f86bfccb7, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 4ed919d2-cc7b-4120-bf73-e8ea0ff719fe, silo_id: 82af3ca8-5091-4869-8146-a9112885b828 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.600483863Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.785497219Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-14T09:51:18.785934401Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.785956841Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.792999952Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-14T09:51:18.785934401Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(f26a90e0-2719-418f-ba7b-06c7da6036f0), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-14T09:51:09.852064397Z }","propolis_id":"f26a90e0-2719-418f-ba7b-06c7da6036f0"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.793587827Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(f26a90e0-2719-418f-ba7b-06c7da6036f0), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-14T09:51:09.852064397Z }, propolis_id: f26a90e0-2719-418f-ba7b-06c7da6036f0, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-14T09:51:18.785934401Z } }","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-14T09:51:18.863083630Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.863118081Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.863138922Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-14T09:51:18.863083630Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(f26a90e0-2719-418f-ba7b-06c7da6036f0), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-14T09:51:09.852064397Z }","propolis_id":"f26a90e0-2719-418f-ba7b-06c7da6036f0"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:18.863151562Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(f26a90e0-2719-418f-ba7b-06c7da6036f0), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-14T09:51:09.852064397Z }, propolis_id: f26a90e0-2719-418f-ba7b-06c7da6036f0, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-14T09:51:18.863083630Z } }","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:20.943445545Z","hostname":"sock","pid":1590,"uri":"//5f617488-2ef8-4cf1-81f7-2b249f5b34c0","method":"GET","req_id":"1a85f77f-08c8-4ac7-abb7-76a9b8e6edd5","remote_addr":"[fd00:1122:3344:101::d]:45846","local_addr":"[fd00:1122:3344:101::1]:51153","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5f617488-2ef8-4cf1-81f7-2b249f5b34c0","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":837,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.698849637Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33779"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.700085668Z","hostname":"sock","pid":1590,"uri":"/instances/1f8794d5-895b-43ae-93ed-692f86bfccb7/state","method":"PUT","req_id":"44402cf3-80de-403f-b31d-d70a40cc9376","remote_addr":"[fd00:1122:3344:101::a]:33779","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1194,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-14T09:51:27.700199451Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.700228922Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.700250253Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-14T09:51:27.700199451Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(f26a90e0-2719-418f-ba7b-06c7da6036f0), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-14T09:51:09.852064397Z }","propolis_id":"f26a90e0-2719-418f-ba7b-06c7da6036f0"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.700278173Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(f26a90e0-2719-418f-ba7b-06c7da6036f0), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-14T09:51:09.852064397Z }, propolis_id: f26a90e0-2719-418f-ba7b-06c7da6036f0, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-14T09:51:27.700199451Z } }","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-14T09:51:27.851186076Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.851224597Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.851303559Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-14T09:51:27.851186076Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-14T09:51:27.851186076Z }","propolis_id":"f26a90e0-2719-418f-ba7b-06c7da6036f0"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.851329259Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.85134034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.851588316Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:27.852172931Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/112d7827-193f-456c-94a5-19163d29475e/debug/bundle/zone\", \"/pool/int/e21e7560-2c98-46a4-bdda-f772e50ec2a3/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/c919f4ac-100e-4c1e-bd24-76220f4848fc/crypt/debug\", \"/pool/ext/9d13b91d-5969-417a-bfea-ad89852618b9/crypt/debug\", \"/pool/ext/39005e7e-189e-49db-8463-a206621e7a54/crypt/debug\", \"/pool/ext/a7771067-edf5-47c9-a972-69320dc9ee62/crypt/debug\", \"/pool/ext/096e5012-bddf-482f-8ad5-65f3b0930676/crypt/debug\", \"/pool/ext/ae2f89f7-35c8-4c05-940c-cdee6d8f8a12/crypt/debug\", \"/pool/ext/42b2209d-f16a-4fca-a84d-54ecc9f13877/crypt/debug\", \"/pool/ext/95ce52af-b115-45d4-acd1-3598b0bafa5d/crypt/debug\", \"/pool/ext/ad348a98-27df-494e-af4b-560a8a421d64/crypt/debug\"] }","zone_name":"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:28.577591887Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0\", bundle_id: 60fb4ce8-86a3-4b8a-8e3d-f794976e1ff4 }, time_created: 2024-05-14T09:51:27.859795567Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_f26a90e0-2719-418f-ba7b-06c7da6036f0","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:28.577694639Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:30.360024986Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:30.388479826Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-14T09:51:27.851186076Z }, propolis_id: f26a90e0-2719-418f-ba7b-06c7da6036f0, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-14T09:51:30.388417655Z } }","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:30.486026839Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34723"}
{"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-05-14T09:51:30.491647693Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:30.491671414Z","hostname":"sock","pid":1590,"uri":"/v2p/49ed10a6-8516-40ff-a573-5f419cb03dfc","method":"DELETE","req_id":"1cb222ce-26e1-403b-80f5-6572a8eb856b","remote_addr":"[fd00:1122:3344:101::c]:34723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":5520,"response_code":"204"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:30.894150362Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-14T09:51:27.851186076Z }, propolis_id: f26a90e0-2719-418f-ba7b-06c7da6036f0, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-14T09:51:30.388417655Z } }","instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:30.898977856Z","hostname":"sock","pid":1590,"instance_id":"1f8794d5-895b-43ae-93ed-692f86bfccb7","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:31.372752094Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41328"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:31.372817916Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"2d985902-28d6-46b6-a7a2-349824e4e39a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41328","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":43,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.609572063Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47283"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.659235727Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c8898bd8-94ea-45a3-ae77-dd31b7bbf690","remote_addr":"[fd00:1122:3344:101::c]:47283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49602,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.65935577Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45736"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.705185316Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a0c3f921-db6c-46ed-adea-9b002a3ffeee","remote_addr":"[fd00:1122:3344:101::a]:45736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":45755,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.705400462Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0b212418-d101-462c-93cf-4c0dcba53169","remote_addr":"[fd00:1122:3344:101::c]:47283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.705613077Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1bfe3946-885d-4d28-8c50-b30d53582642","remote_addr":"[fd00:1122:3344:101::a]:45736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":132,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.728360991Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60425"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.773688804Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e2df5cb8-5775-471b-bce0-8d23b65e75ae","remote_addr":"[fd00:1122:3344:101::b]:60425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":45280,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-14T09:51:33.773982102Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d92334be-99be-429c-a1ff-6befd07fc546","remote_addr":"[fd00:1122:3344:101::b]:60425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":146,"response_code":"200"}