[ Jul 26 02:33:54 Enabled. ] [ Jul 26 02:33:54 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jul 26 02:33:54 Method "start" exited with status 0. ] [ Jul 26 02:33:54 Rereading configuration. ] [ Jul 26 02:33:54 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:54.136099769Z","hostname":"sock","pid":1555,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:54.136250912Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:54.190493419Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:54.1905437Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:54.305217668Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:54.314182677Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:54.322129223Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:54.927408097Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.069521145Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:125"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.069549986Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:137"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.069631708Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:150"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.069698229Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:172","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.06970989Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.06971906Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.072772761Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:33:55.072778681Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.072793162Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:186"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.072797082Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.072811142Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.072894654Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.072959405Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.072983106Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.072996426Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.084612928Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.084654249Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.184664224Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.398341424Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.398379275Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.398387625Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.398398955Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.398405835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.398412005Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.398432076Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.407502038Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.407525578Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.502623839Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711078157Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711123768Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711159189Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711165369Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711170279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711176919Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711181709Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711191429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71119913Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71121294Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71121777Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711232Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711235621Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71124007Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711244691Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711248691Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711263591Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_3.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711268301Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711272401Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711275591Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711291522Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711295602Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711307532Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711313322Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711359903Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711369164Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711374124Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711383684Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711387874Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711406905Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711413355Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711417605Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711421685Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711427205Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711438695Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711443395Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711457006Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711474746Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711481056Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711484366Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711488836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711494067Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711498187Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711501847Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711529677Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711534268Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711540688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711544418Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711548538Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:96"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711558798Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:98"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711562238Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711594979Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711600099Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711603629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711607159Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:210"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711610619Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71164217Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71164653Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71165019Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71165867Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.71166234Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.7116686Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711675581Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711679611Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711705681Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:215"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.711711052Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:230"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.722029682Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:722"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.722062943Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.999923148s","next_cleanup":"Instant { tv_sec: 12794, tv_nsec: 811918527 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.722103594Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3661"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.722158256Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.722206757Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.722214197Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725497923Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725578575Z","hostname":"sock","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.72578417Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1017"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.72579232Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725839871Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725890223Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725911793Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725920203Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1077"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725943974Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725962214Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725968665Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725974455Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1118"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725977575Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.725983535Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.726012026Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.726041356Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.726381774Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.726658761Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.726670481Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1151"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.726812084Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.726819564Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.726975198Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727111511Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727169322Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ea0e0fc4-fc93-4de6-b687-98567d824e83, rack_id: 6101b963-ee1b-430c-ad60-5a5922d708b7, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727296476Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55089","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ea0e0fc4-fc93-4de6-b687-98567d824e83, rack_id: 6101b963-ee1b-430c-ad60-5a5922d708b7, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727407598Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727413398Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727442089Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727455009Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:573"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.72747941Z","hostname":"sock","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.72748386Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.72749166Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.72750079Z","hostname":"sock","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727506661Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.72751285Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:562"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727516051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:590"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727520971Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:394"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727532681Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.727540471Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:480","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.729515907Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3895"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.729569529Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4268"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.760271646Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.781225975Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.798542479Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.845888705Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:368"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.845940876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.845947686Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:380"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.845953356Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 134.737258ms)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.849227953Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"96da0c5b-a072-4adf-b782-dcacd1638aad\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:33:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"96da0c5b-a072-4adf-b782-dcacd1638aad\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.859878872Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.883692378Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.883737109Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.89064882Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:395"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.89065721Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.89066229Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.963570243Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.984640564Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 90.877879ms)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:55.98531822Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"16eaa63a-1367-494e-840e-6fb2c88dc57f\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:33:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"16eaa63a-1367-494e-840e-6fb2c88dc57f\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.07738935Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 73.713032ms)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.077908462Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7f01bd36-8f22-4bbb-8b63-7d50c0361984\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:33:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7f01bd36-8f22-4bbb-8b63-7d50c0361984\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.153252802Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 252.194671ms)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.153735423Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"69b17e8b-cc59-457f-9f39-872ab32ae78b\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:33:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"69b17e8b-cc59-457f-9f39-872ab32ae78b\" }"} {"msg":"Mounting oxi_2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.183381695Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.19643855Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.196520852Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.196538792Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.196567263Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.407447927Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 223.787058ms)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.408545713Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5dee47de-f41c-42fe-8135-27bbfb42e2d9\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:33:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5dee47de-f41c-42fe-8135-27bbfb42e2d9\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.633404794Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 212.675498ms)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.634161482Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"76ed8039-551b-49ac-a0a2-aaeafb8669d6\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:33:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"76ed8039-551b-49ac-a0a2-aaeafb8669d6\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.848229331Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 420.906112ms)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.848813524Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef5d251e-3d1d-49f4-826a-a7ac746f6946\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:33:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef5d251e-3d1d-49f4-826a-a7ac746f6946\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.921523922Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.922147417Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.922179097Z","hostname":"sock","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.922224208Z","hostname":"sock","pid":1555,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:33355","producer_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:33:56.923530499Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"157.085485ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.968297983Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.968335024Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.968407486Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:905","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.968415616Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969201225Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969229045Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969256636Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969270466Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969492751Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969656285Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969692166Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969703156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3661"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:56.969723027Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55089","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.137649248Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.1377127Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.137993056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.138093839Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.138120099Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.138188091Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.138225912Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:33:57.1389882Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"465.983532ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:33:57.139099072Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/nexus.rs:600"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.178322188Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34661"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.178493532Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.178576994Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"255c2327-a22c-4074-a20c-36482ac831f9","remote_addr":"[fd00:1122:3344:101::1]:34661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.209596288Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.20967414Z","hostname":"sock","pid":1555,"uri":"/sled-role","method":"GET","req_id":"d79449f5-1737-4f0c-8e7b-1ae4e1b6f07b","remote_addr":"[fd00:1122:3344:101::1]:42948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":25,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.211121714Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.211147954Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.211522823Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.211760929Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:830"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 11f51cfd-88b7-4f18-8951-7d757e18895b, pool_id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: ce0e8de8-ec3e-4a65-9b12-4b3d05e2af7c, pool_id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 9426f29b-4ad6-432a-9152-3ddfd0b60d09, pool_id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 6132d37c-1434-4fed-a5c9-ade469b740a0, pool_id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 6c848580-6df8-44cd-b026-f3c062e46850, pool_id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 88274dc3-ce1f-4b2a-9966-f91952efd323, pool_id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: e10e4f5f-4a5f-47ac-ad38-15eb242bc36f, pool_id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 3b3f1b92-8896-4279-9dde-6984c541854b, pool_id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 20f5fcb2-ea2f-4860-9d58-9d8a18d63e8b, pool_id: 3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.240855678Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:355"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.240999542Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:44735"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241146295Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:827"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 11f51cfd-88b7-4f18-8951-7d757e18895b, pool_id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: ce0e8de8-ec3e-4a65-9b12-4b3d05e2af7c, pool_id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 9426f29b-4ad6-432a-9152-3ddfd0b60d09, pool_id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 6132d37c-1434-4fed-a5c9-ade469b740a0, pool_id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 6c848580-6df8-44cd-b026-f3c062e46850, pool_id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 88274dc3-ce1f-4b2a-9966-f91952efd323, pool_id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: e10e4f5f-4a5f-47ac-ad38-15eb242bc36f, pool_id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 3b3f1b92-8896-4279-9dde-6984c541854b, pool_id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 20f5fcb2-ea2f-4860-9d58-9d8a18d63e8b, pool_id: 3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241167055Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241177176Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241184736Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241189536Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241245287Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241251508Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241259458Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.241263688Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.321388729Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.330504292Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:57.330522282Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:33:57.711817846Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"620.557678ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:33:57.970983297Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:58.317233373Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:33:58.478864027Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.916927833s"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:58.484641522Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:58.484720694Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:58.484732234Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:58.484740324Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:58.484748005Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:58.484842627Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:58.484849137Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:33:59.021060248Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:33:59.021255532Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/nexus.rs:600"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.023397102Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.209838826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2472"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.209876027Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2690"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.209887177Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2342"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.209904898Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2389"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.209984719Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2771"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.20998886Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2813"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.212425136Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2615"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.214192718Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2383"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.214328281Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.214488194Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Automatically destroying dataset: oxp_4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:33:59.372914664Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 70.800944ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:33:59.978221198Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:00.384814283Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 74.974251ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:00.608345643Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 235.267469ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:01.347168175Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:01.350005371Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/nexus.rs:600"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 220.677335ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:01.998478703Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:02.460670886Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.403578651s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:02.460921632Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 737.195215ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:02.461102326Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:02.704116511Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:02.920265548Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:02.92032664Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:02.92033474Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:02.92034225Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:02.92034859Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:03.155955242Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:03.156013233Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:04.354782805Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:04.608143012Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:04.608203803Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:04.608211043Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:04.608224654Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:04.608233624Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:04.608398018Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:04.608423859Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:05.145801187Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/nexus.rs:600"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:05.145923729Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:05.85122732Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3919","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:05.851315151Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3935"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:05.851320272Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:518"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:06.075323112Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:06.304633727Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Automatically destroying dataset: oxp_b8ef9471-5b81-40ed-90b5-1103a4a9eb91/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:06.310732249Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:06.788202329Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/nexus.rs:600"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:07.030574898Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"7.220367583s"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.486555075Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:07.488208094Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Looking for unencrypted datasets in oxp_b8ef9471-5b81-40ed-90b5-1103a4a9eb91","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.569346279Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.576784163Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.576817683Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.576826294Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.576834494Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.576841464Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.585605889Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.585632979Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"a47af5d0-e565-4513-84ad-c553b7265a72 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.68201643Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.690028217Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.690074058Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.690081188Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.690129249Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.690192771Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.690329604Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.690336454Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.726462828Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a47af5d0-e565-4513-84ad-c553b7265a72/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.734469665Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_a47af5d0-e565-4513-84ad-c553b7265a72","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.858839259Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.867211985Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.867249225Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.867258386Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.867266746Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.867290257Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.877671039Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.877702569Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.975679587Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.983246484Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.983286444Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.983301445Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.983349206Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.983360326Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.983493529Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:07.983500139Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.018732252Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2a33f7b1-2bfd-4239-9138-fb534f4ccb3f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.027271032Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_2a33f7b1-2bfd-4239-9138-fb534f4ccb3f","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.151559844Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.160004131Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.160051322Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.160061592Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.160071323Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.160077543Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.169670657Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.169703898Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.264157873Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.272020447Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.272112779Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.272122499Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.27216361Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.272195201Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.272360435Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.272369545Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.307663419Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a38a1ea7-44a1-411f-bbb8-767c9a23135f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.315770989Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_a38a1ea7-44a1-411f-bbb8-767c9a23135f","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.441250839Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.449077021Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.449114252Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.449120112Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.449127363Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.449131653Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.458714936Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.458747627Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"cb102b97-0352-4c03-8d37-4c80c4475510 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.554477523Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.562379517Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.562423238Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.562429458Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.562479359Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.56248881Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.562619143Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.562630423Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:08.566692748Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.600147829Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cb102b97-0352-4c03-8d37-4c80c4475510/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.607979742Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:08.612150799Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/nexus.rs:600"} {"msg":"Looking for unencrypted datasets in oxp_cb102b97-0352-4c03-8d37-4c80c4475510","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.736474593Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.7444768Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.744516001Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.744524471Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.744533621Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.744582092Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.754044363Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.754085894Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.853757632Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.861888861Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.861932302Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.861938563Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.861979753Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.862004004Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.862175948Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.862185278Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.897457032Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c5566b16-9a9f-498b-a4b1-83ac43308db5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:08.905280344Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_c5566b16-9a9f-498b-a4b1-83ac43308db5","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.027670502Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.035592747Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.035627178Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.035633258Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.035638588Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.035642978Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.045491568Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.045525199Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"136ed548-ab63-44da-a187-0fa13dc111a0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.14406397Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.154589096Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.154631437Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.154639707Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.15477085Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.15478295Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.154891333Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.154902053Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.195354778Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_136ed548-ab63-44da-a187-0fa13dc111a0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.20315916Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_136ed548-ab63-44da-a187-0fa13dc111a0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.329172843Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.336812621Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.336850872Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.336871622Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.336878722Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.336884173Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.346468746Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.346498037Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.442221542Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.450401893Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.450433744Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.450439965Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.450465815Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.450477185Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.450617129Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.450624299Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.487629703Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.496452779Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:09.569908294Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Looking for unencrypted datasets in oxp_3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.627368296Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.635448735Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.635488166Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.635553137Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.63566687Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f09c5312-7869-4743-8a0d-863eb3bf0156 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.635750542Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.635759252Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.635851684Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.636061769Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.636080189Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.636085989Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:857","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.63609401Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:862"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.63610107Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:867"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.63611923Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:116","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.636127901Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:873"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.636194972Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:822","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.636212452Z","hostname":"sock","pid":1555,"sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:878"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.636285574Z","hostname":"sock","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"665f7da0-e4dc-41e9-9c56-88ff6ac7ee13","remote_addr":"[fd00:1122:3344:101::1]:44735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":12395250,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.666696304Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.666849118Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:52523"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.667185086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.667194446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.720369908Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.720421269Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.769238509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.76927317Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.819053752Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.819088133Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.819096033Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.819150015Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.863396308Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.863435659Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.863439899Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.914103442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.917682225Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.921395912Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.933964156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.938343218Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:09.941966672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:10.572372063Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ea0e0fc4-fc93-4de6-b687-98567d824e83): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:10.611368193Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/nexus.rs:600"} {"msg":"Profile for oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:11.403696515Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:11.403769046Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:11.883420727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:11.883496289Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:11.887448551Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:12.367730916Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:12.367786487Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 54.895447ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.381382325Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:12.381442906Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:12.381524188Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 92.735536ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.444942179Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40\") failed: Property not found. retry in 35.009631ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.492798776Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 29.078247ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.509620589Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40\") failed: Property not found. retry in 137.551517ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.535426672Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 166.654458ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.545237501Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 148.233182ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.551740683Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40\") failed: Property not found. retry in 288.670036ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.71691016Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 284.010617ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.780180838Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 410.349169ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:12.812192325Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40\") failed: Property not found. retry in 526.575754ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:13.018845291Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 400.636883ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:13.077695875Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 707.414186ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:13.232544931Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 635.643165ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:13.4868818Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40\") failed: Property not found. retry in 1.078079327s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:13.554781616Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 833.988866ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:13.950692451Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 1.363184546s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:14.134886041Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40\") failed: Property not found. retry in 702.161213ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:14.641177714Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 1.255473366s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:14.79343794Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40\") failed: Property not found. retry in 1.33036558s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:15.350889307Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 813.825333ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:15.50519329Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.675288892Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3943"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.675338593Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.680898483Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4032"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.805863011Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4080"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.805901732Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4107"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.844120174Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.844157595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3984"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.897952612Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4026"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.897987703Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4087","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.917482608Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4098"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:15.917508768Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4140"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.004214073Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.004256654Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4178"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 1.097771799s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:16.057923797Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.119090636Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4219"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.119156917Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"Failed to get switch ID from MGS (retrying in 55.901202ms)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:16.151485072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:383","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Failed to get switch ID from MGS (retrying in 64.744537ms)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:16.208544284Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:383","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Failed to get switch ID from MGS (retrying in 239.518105ms)","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:16.273706496Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:383","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 561.947656ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:16.328294041Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.515699367Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.546964157Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.54707072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.547547871Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.547577852Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.585444946Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.606330713Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625633554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625727017Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625752677Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625761017Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625768147Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625776878Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625784108Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625791978Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625800708Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625807498Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625815708Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625822509Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625830749Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625839139Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.62585964Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.62586503Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.62587004Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.62587509Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.62588018Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625885Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.62588993Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625894741Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:16.625899681Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40\") failed: Property not found. retry in 914.115801ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:16.690600152Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b\") failed: Property not found. retry in 1.201890722s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:16.898318312Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874\") failed: Property not found. retry in 734.672578ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:17.166561525Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.118970566Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_internal_dns_25724e7e-f0f3-413d-ab41-b4a33973050b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.119031987Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_internal_dns_4e64b0cb-e3e2-4959-adf4-d5cd70f25b40"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.119040357Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_internal_dns_82379658-84e2-487b-bcd4-b30ffc1d9874"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.119052228Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.119354775Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.119650452Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"57753561-9544-4d23-ab2b-bc0defcbb7ff","remote_addr":"[fd00:1122:3344:101::1]:52523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8452713,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.121494575Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.166921235Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.213655067Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.260830428Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:591"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.260843239Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.29084944Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.292706763Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.292720173Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.322205022Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.322437467Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57497"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.322784245Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.322794735Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.340840597Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.340897478Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.355811666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.355853407Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.37095514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.371003681Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.371171435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.371294968Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.382698764Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.414703512Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.415391848Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.415412858Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.444735273Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 143, 87]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.444751593Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.445239505Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.445246825Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.445598443Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.445999772Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.470527605Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:18.491108056Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:19.130586578Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:19.1306882Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965\") failed: Property not found. retry in 47.987927ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:19.296051862Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965\") failed: Property not found. retry in 115.677211ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:19.358749336Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965\") failed: Property not found. retry in 218.068044ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:19.499637036Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965\") failed: Property not found. retry in 354.678647ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:19.727308533Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965\") failed: Property not found. retry in 430.92825ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:20.094331003Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965\") failed: Property not found. retry in 1.313211245s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:20.53399182Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965\") failed: Property not found. retry in 1.310893582s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:21.856604914Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965\") failed: Property not found. retry in 931.291802ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:23.177224231Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.131274949Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.131335061Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.131681779Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.131990466Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"215b99fa-1563-4eca-927b-cfb1230d8d7d","remote_addr":"[fd00:1122:3344:101::1]:57497","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":5809479,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.133730397Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.167093146Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.16728817Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59646"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:24.176716681Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.176838733Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"42651165-ac85-4d3f-a783-ab5a111aca6c","remote_addr":"[fd00:1122:3344:101::1]:59646","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9397,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:24.180313365Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.265853152Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:24.265971825Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:44637"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:26.273544773Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:26.273639266Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"6c452739-0f92-44c8-89cd-9794657f60db","remote_addr":"[fd00:1122:3344:101::1]:44637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":2007550,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:26.276368069Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:26.441894735Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:26.44211971Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:52868"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:26.452219156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:26.452344909Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"9e57eb4e-92f4-4320-bd1f-d224a06dc569","remote_addr":"[fd00:1122:3344:101::1]:52868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":10168,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:26.454684833Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:26.617468175Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:26.617588147Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38120"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0eb4388e-1e4e-49da-b41b-58746266c965': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-07-26T02:34:30.626533911Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:30.626622283Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"41a23b36-8d29-43df-8453-53417e3e51f7","remote_addr":"[fd00:1122:3344:101::1]:38120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008918,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:30.629830837Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:30.816782663Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:30.816932027Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:45930"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:30.825175199Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"637425ba-a588-46a8-925e-1f098753b1f5","remote_addr":"[fd00:1122:3344:101::1]:45930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8112,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:30.828014665Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:30.828024506Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.05565598Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.055794113Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.076539568Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"176c579e-3b3a-4e71-b3d9-e56002ae1dd2","remote_addr":"[fd00:1122:3344:101::1]:34251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":20590,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3330918944, ip_addr: 198.137.202.32, stratum: 3, ref_time: 1721961270.6833196, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.080283985Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.080293326Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1230"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.110754927Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.11090274Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:52946"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.111225158Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.111234848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.131974132Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.132015893Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.150135577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.150168747Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.168876034Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.168906925Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.221699337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.221724988Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.280176163Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:31.280229244Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.013576717Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.013625148Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.021348139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.02139681Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.073929087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.073979418Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.074195443Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.074342906Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.106482737Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.106536018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.106545458Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.106552009Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.106558458Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.106671921Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b7b8f59c-dce4-4936-9e3f-8c03b0b5572d","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized)]","latency_us":361696,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.106856276Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"57.957421ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7b8f59c-dce4-4936-9e3f-8c03b0b5572d\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b7b8f59c-dce4-4936-9e3f-8c03b0b5572d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.166306724Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.166639001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.166672482Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.185056142Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.185100723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.202582331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.202617662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.220022818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.220069719Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.23724248Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.237285341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.256026709Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.25607157Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.274958511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.274999472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.292785097Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.292823898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.310381858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.31044225Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.310621044Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.310772267Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.341173907Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.341219388Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.341229309Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.341239089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.341266979Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.341387312Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"45bf417b-cf26-481d-8388-dbba3ea42290","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized)]","latency_us":174936,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.341540336Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"164.887512ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"45bf417b-cf26-481d-8388-dbba3ea42290\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"45bf417b-cf26-481d-8388-dbba3ea42290\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.507948431Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.50835412Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.50836798Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.527845645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.527890707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.545352364Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.545385615Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.561503111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.561544132Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.579707977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.579742337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.596441987Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.596474368Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.614751375Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.614784405Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.63295999Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.632999071Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.64965249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.649701671Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.649951977Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.650136031Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.680339136Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.680380517Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.680390397Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.680398138Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.680407838Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.68051406Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7fde6d93-3c8f-4e89-9295-3bad561f0436","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized)]","latency_us":172371,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.680672824Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"85.587557ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7fde6d93-3c8f-4e89-9295-3bad561f0436\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7fde6d93-3c8f-4e89-9295-3bad561f0436\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.766814846Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.767185164Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.767208805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.785603745Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.785640225Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.802865888Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.802906169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.821504783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.821555244Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.83851866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.838550491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.85607646Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.856114591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.87363894Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.873670401Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.891683942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.891734173Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.909867676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.909927148Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.910141053Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.910291446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.941168507Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.941202368Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.941208688Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.941215218Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.941223259Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:32.941363432Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e1d55137-82c1-40e5-a564-077a91d1104e","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized)]","latency_us":174385,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:32.941516855Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"96.243026ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1d55137-82c1-40e5-a564-077a91d1104e\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e1d55137-82c1-40e5-a564-077a91d1104e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.038583374Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.038977144Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.038988334Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.057952918Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.057988599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.07596275Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.07599076Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.095747693Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.095799584Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.113774395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.113799585Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.131552521Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.131588942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.151015517Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.151045048Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.168916746Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.168945017Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.186471207Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.186520518Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.186772914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.186895667Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.21988458Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.21992247Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.219930921Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.219939221Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.219945901Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.220061744Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b77943c8-30e8-4639-8a39-a36ac86c4a7f","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized)]","latency_us":181794,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.220224448Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"276.568585ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b77943c8-30e8-4639-8a39-a36ac86c4a7f\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b77943c8-30e8-4639-8a39-a36ac86c4a7f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.496850884Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.49711704Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.497127291Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.518291196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.518337177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.536938683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.537000434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.554951394Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.554988735Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.572475656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.572523786Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.590548618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.590589599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.608654392Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.608698873Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.626365977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.626410508Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.644736877Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.644785148Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.645003903Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.645155726Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.67692216Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.676973102Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.676983332Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.676993782Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.677001652Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.677134455Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"cf5af7c3-4f2d-4e10-8d60-be59d292e426","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized)]","latency_us":180661,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:33.677300889Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"266.679497ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cf5af7c3-4f2d-4e10-8d60-be59d292e426\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cf5af7c3-4f2d-4e10-8d60-be59d292e426\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.943979103Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.944375583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.944402784Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.963442039Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.96348894Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.980087299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.98012054Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.996208457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:33.996250378Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.014077375Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.014112326Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.030875608Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.030919339Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.049981176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.050018337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.068015038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.068046979Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.083740416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.083781837Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.084003072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.084164406Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.115425288Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.115465689Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.115476199Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.115491919Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.11550172Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.115606442Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1c2a28d7-84a5-4b4c-85e3-31de2dc00d9a","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized)]","latency_us":171915,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.115762126Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"292.151585ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1c2a28d7-84a5-4b4c-85e3-31de2dc00d9a\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1c2a28d7-84a5-4b4c-85e3-31de2dc00d9a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.408419718Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.408801038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.408813538Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.428056678Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.42811299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.447026642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.447068763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.465186758Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.465222339Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.482645167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.482677697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.500244718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.500286659Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.517682476Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.517725687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.535386251Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.535423731Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.552176024Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.552220925Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.5524177Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.552568393Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.584110822Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.584169683Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.584179943Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.584191144Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.584200994Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.584281526Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6c703f39-bcbc-4e1f-acbc-2b7b6946149b","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized)]","latency_us":176178,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.584392878Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"286.264619ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6c703f39-bcbc-4e1f-acbc-2b7b6946149b\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6c703f39-bcbc-4e1f-acbc-2b7b6946149b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:34.813168315Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"306.321315ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.871146353Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.871427309Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.871437059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.89026604Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.890307271Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.907566625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.907600336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.924989273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.925027774Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.942551704Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.942587225Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.96158607Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.961614891Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.979213813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.979257564Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.996901067Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:34.996940708Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.014579795Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.014618992Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.014804776Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.01496683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.047523973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.047556513Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.047565143Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.047573514Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.047580304Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.047697446Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"04e9b92a-cfff-4880-b849-282700118d55","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized)]","latency_us":176917,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.04786173Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"384.254657ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"04e9b92a-cfff-4880-b849-282700118d55\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"04e9b92a-cfff-4880-b849-282700118d55\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.432373264Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.432741013Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.432752863Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.451612684Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.451652815Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.469159775Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.469202226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.486577183Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.486606014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.504165354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.504197885Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.522694268Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.522720789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.540579437Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.540611688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.558326442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.558370943Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.575961045Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.576000756Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.576240082Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.576403936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.608490327Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.608539328Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.608549028Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.608558529Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.608568089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:35.608651911Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"524e999d-7a1c-47c1-b2f3-bdeb60771e81","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized)]","latency_us":176595,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:35.608775654Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"682.481924ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"524e999d-7a1c-47c1-b2f3-bdeb60771e81\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"524e999d-7a1c-47c1-b2f3-bdeb60771e81\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.290413915Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.290744472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.290756723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.31072622Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.310765641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.32867377Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.328707681Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.346381245Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.346426416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.364196352Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.364231673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.381945288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.381974328Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.399695173Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.399732434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.417602413Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.417639243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.435769328Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.435802319Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.435990623Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.436158227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:36.466778654Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:36.466838926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:36.466850586Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:36.466859886Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:36.466869576Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.466969298Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"66382181-ca96-469f-9dbf-e356307c44ac","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized)]","latency_us":176889,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:36.467122232Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"496.701775ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"66382181-ca96-469f-9dbf-e356307c44ac\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"66382181-ca96-469f-9dbf-e356307c44ac\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.96331333Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.964389545Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.964402495Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.985274354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:36.985326895Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.004279069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.00432625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.023475658Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.02353811Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.04022346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.040273962Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.057348261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.057405263Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.076567111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.076601412Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.094043671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.094081112Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.111475709Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.111509149Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.111708374Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.111846259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.142968836Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.143005757Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.143014457Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.143021327Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.143028067Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.14313716Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"132101db-1537-4db0-89f9-e175fb706594","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized)]","latency_us":180161,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.143404186Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"395.408957ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"132101db-1537-4db0-89f9-e175fb706594\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"132101db-1537-4db0-89f9-e175fb706594\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.538314183Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.538565499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.538589069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.559235963Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.559284874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.577172343Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.577215593Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.594747164Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.594791495Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.613259928Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.613297938Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.631260459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.63130462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.649107947Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.649142278Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.665918981Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.665955622Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.684725961Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.684776902Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.684959616Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.68510155Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.715955792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.716015883Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.716027524Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.716056495Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.716065125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:37.716149547Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bd178f74-0d03-43cb-960c-5e5d4a2f5b47","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized)]","latency_us":178210,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:37.71630977Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"548.273299ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bd178f74-0d03-43cb-960c-5e5d4a2f5b47\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bd178f74-0d03-43cb-960c-5e5d4a2f5b47\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.263220168Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.263439302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.263460253Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.282360056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.282404587Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.300263115Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.300293576Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.3179928Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.318023401Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.335730606Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.335763976Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.353715597Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.353741187Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.371401675Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.371435252Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.389621158Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.389655809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.408361916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.408407888Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.408597742Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.408727895Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:38.441193882Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:38.441239686Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:38.441248817Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:38.441259407Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:38.441269077Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:38.44138754Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1a09820d-8c69-486b-a91e-35cf73efb966","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized)]","latency_us":178586,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:38.441590265Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"787.22934ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1a09820d-8c69-486b-a91e-35cf73efb966\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1a09820d-8c69-486b-a91e-35cf73efb966\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.22795498Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.228317807Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.228345728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.248863528Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.248907743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.267351191Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.267410653Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.285963887Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.286000398Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.304215594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.304254735Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.322031212Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.322065403Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.339608393Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.339647147Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.35741822Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.357453061Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.376659861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.376706912Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.376905807Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.37706677Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:39.409055459Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:39.40909522Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:39.409105321Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:39.409115211Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:39.409125161Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:39.409208803Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"724dbf5a-5454-4442-86ea-3c4ae0e374d9","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized)]","latency_us":181598,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:39.409367367Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"788.904792ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"724dbf5a-5454-4442-86ea-3c4ae0e374d9\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"724dbf5a-5454-4442-86ea-3c4ae0e374d9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.197769137Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.199032826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.199041526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.219492675Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.219541326Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.237763143Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.237805374Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.255108989Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.255160651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.272500057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.272534527Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.289010433Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.289048594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.306483732Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.306522703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.325120989Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.325152219Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.341765738Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.34182133Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.342025525Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.342166408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:40.374209018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:40.374256109Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:40.374265909Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:40.374275449Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:40.37428169Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:40.374395962Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b2d9486f-6f96-4ce5-884c-f933786f627b","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized)]","latency_us":176906,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:40.374551536Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.446892908s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b2d9486f-6f96-4ce5-884c-f933786f627b\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b2d9486f-6f96-4ce5-884c-f933786f627b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.818615347Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.819511778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.819523799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.840977491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.841007402Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.861750347Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.861782668Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.880537197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.880577128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.899920331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.899959392Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.919611512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.919656373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.937813829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.937851369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.956738802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.956771052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.976484254Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.976529125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.9767535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:41.976928534Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:42.009600329Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:42.00963843Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:42.00964708Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:42.009653761Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:42.009661601Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:42.009775014Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d210d2ce-b3f1-4e19-81ca-5ee5a1c63bca","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized)]","latency_us":191542,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:42.009934597Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.42801443s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d210d2ce-b3f1-4e19-81ca-5ee5a1c63bca\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d210d2ce-b3f1-4e19-81ca-5ee5a1c63bca\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.432638634Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.433088715Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.433115605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.452564131Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.452619972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.47222977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.472265131Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.490610591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.490646625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.509768229Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.50982002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.52816937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.528210281Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.546188172Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.546238113Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.564811288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.564849449Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.582528253Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.582576184Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.58283073Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.582988544Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:44.616391926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:44.616432827Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:44.616446927Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:44.616454977Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:44.616461848Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:44.61655936Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7b07c8fb-cae8-4a15-8fa1-3297799e8014","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized)]","latency_us":184230,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:44.616736044Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"3.202822289s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7b07c8fb-cae8-4a15-8fa1-3297799e8014\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7b07c8fb-cae8-4a15-8fa1-3297799e8014\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:46.393509167Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:46.393696502Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"028bc0ed-f281-4076-8f0e-f5a09f4d5165","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42950","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":79,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.811419026Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.811827076Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.811846576Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.833421842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.833465513Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.85128577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.851318051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.86965556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.869689241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.886585717Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.886619557Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.903779149Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.90381261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.920670595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.920703976Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.937447968Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.937472948Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.954120968Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.954164559Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.954336603Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.954484987Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:47.985674807Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:47.985720778Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:47.985728628Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:47.985735028Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:47.985741238Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:47.985856781Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e4c5525e-b052-48a0-a0ed-08c95c1f0d71","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized)]","latency_us":174720,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:47.986064406Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.094100738s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e4c5525e-b052-48a0-a0ed-08c95c1f0d71\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e4c5525e-b052-48a0-a0ed-08c95c1f0d71\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.075230083Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.075614422Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.075625592Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.09604404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.096080581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.113244223Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.113272044Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:50.118998668Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"368.218682ms"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.132062704Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.132085994Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.1506965Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.150724931Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.167542374Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.167574485Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.185612318Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.185632068Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.203790423Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.203821124Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.220977756Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.221022109Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.221184251Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.221314964Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:50.253135209Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:50.25318741Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:50.25320511Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:50.253213431Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:50.253223611Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:50.253336294Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bae4f230-491e-47fb-b101-29bb8a854b9d","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized)]","latency_us":178454,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:50.253517248Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"3.990933601s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bae4f230-491e-47fb-b101-29bb8a854b9d\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bae4f230-491e-47fb-b101-29bb8a854b9d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.234911671Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.235242809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.235256629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.258073468Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.258112364Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.276970936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.27700494Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.295553831Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.295576482Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.314258259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.314287249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.331866201Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.331889502Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.349656728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.349685588Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.367325191Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.367353582Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.38475599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.384799991Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.385000875Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.385134299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:54.416236567Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:54.416278808Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:54.416286528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:54.416296098Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:54.416305708Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:54.416408631Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e75ed8d5-0f05-4d4b-b18a-bc7d511fb64b","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized)]","latency_us":181874,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:54.416573444Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.886456722s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e75ed8d5-0f05-4d4b-b18a-bc7d511fb64b\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e75ed8d5-0f05-4d4b-b18a-bc7d511fb64b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.291594681Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.292918902Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.292927592Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.313517844Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.313570205Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.331997057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.332046368Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.349545848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.349581299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.367844236Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.367888647Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.385968541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.386004652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.403994233Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.404040756Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.421445211Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.421484902Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.438759016Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.438806987Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.439038053Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.439188726Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:59.470178412Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:59.470213352Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:59.470223793Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:59.470233393Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:59.470243083Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:34:59.470358856Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"981bb968-8968-4885-9f36-6c8a37cd7886","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized)]","latency_us":179126,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:34:59.470511939Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"6.526941977s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"981bb968-8968-4885-9f36-6c8a37cd7886\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:34:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"981bb968-8968-4885-9f36-6c8a37cd7886\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:05.482319083Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.391043226s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:05.981184515Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:05.98186312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:05.981870391Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.005448083Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.005495584Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.023152047Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.023196788Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.04079886Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.040841612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.060163304Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.060205625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.079745742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.079783583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.099789002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.099821569Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.118682384Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.118711705Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.136250775Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.136300707Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.136516922Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.136661405Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:06.168388328Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:06.168422799Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:06.168433679Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:06.168442079Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:06.16845002Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:06.16863323Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"fba67079-fe87-4066-bc30-356b3c7df1c5","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized)]","latency_us":187888,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:06.168853489Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"6.823457524s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fba67079-fe87-4066-bc30-356b3c7df1c5\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:35:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fba67079-fe87-4066-bc30-356b3c7df1c5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:12.975028352Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:12.975346369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:12.97536443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:12.995838349Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:12.99587317Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.017459886Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.017504747Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.035846696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.035899037Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.055320912Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.055362123Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.074660965Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.074686786Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.093090297Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.093123657Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.111150889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.11118578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.128949976Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.129000457Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.129281664Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.129425577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:13.161365415Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:13.161408456Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:13.161418406Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:13.161428287Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:13.161437787Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:13.161533349Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8a83e676-25a2-4f88-9555-6f718836b2ab","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized)]","latency_us":186905,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:13.161685763Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"5.165478931s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a83e676-25a2-4f88-9555-6f718836b2ab\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:35:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a83e676-25a2-4f88-9555-6f718836b2ab\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:16.345301276Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41568"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:16.3454737Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"24a7a00c-f367-4367-a1a8-18563e65e77b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41568","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":81,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.315111828Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.315365854Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.315400595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.336043898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.336083069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.354160512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.354193023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.373506555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.373554766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.391278151Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.391316892Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.409043887Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.409076288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.426023745Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.426063085Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.444266052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.444304882Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.461437674Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.461478515Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.46169416Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.461851893Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:18.492274136Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:18.492306406Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:18.492316447Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:18.492325717Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:18.492334487Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:18.492491441Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"62766a10-0ace-4f9f-a0d5-154e72e04005","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized)]","latency_us":177778,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:18.492693286Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.652766869s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"62766a10-0ace-4f9f-a0d5-154e72e04005\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:35:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"62766a10-0ace-4f9f-a0d5-154e72e04005\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:21.865490639Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"2.361278625s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.133071429Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.133389717Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.133403247Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.155220068Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.15528052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.173957047Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.173992678Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.192986842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.193038133Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.211552057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.211595918Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.230840999Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.23088018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.24882024Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.24885856Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.267291372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.267330543Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.286090143Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.286144564Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.286355099Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.286486362Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:23.318495211Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:23.318536112Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:23.318543782Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:23.318550423Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:23.318556773Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:23.318664895Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"212506e8-041b-419e-a689-e560b2b82799","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized)]","latency_us":185977,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:23.318835429Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"9.781757648s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"212506e8-041b-419e-a689-e560b2b82799\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:35:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"212506e8-041b-419e-a689-e560b2b82799\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.075954979Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.077091136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.077097556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.098890096Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.098909017Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.118602818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.118633899Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.137277915Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.137309866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.156531506Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.156557117Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.174705612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.174722342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.192725074Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.192753284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.211666487Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.211695148Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.229520295Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.229564456Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.229828162Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.229983371Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:33.262687902Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:33.262717523Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:33.262723523Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:33.262729063Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:33.262736233Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:33.262845146Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"fcc37c55-72df-4dd7-9e5f-0e4c291a0aee","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized)]","latency_us":187303,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:33.263008959Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"10.700510248s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fcc37c55-72df-4dd7-9e5f-0e4c291a0aee\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:35:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fcc37c55-72df-4dd7-9e5f-0e4c291a0aee\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:39.218435364Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"4.94102194s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.936049025Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.936336452Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.936348092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.958743647Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.958791645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.977038065Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.977072186Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.995279152Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:43.995322773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.013356935Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.013403627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.031816418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.031856959Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.050051735Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.050087806Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.06823888Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.068273001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.088311361Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.088363252Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.088629338Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.088758741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:44.121112458Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:44.121153799Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:44.12116031Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:44.12116442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:44.12116963Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:44.121273302Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f0877265-abb8-4724-973b-3d5714310a6b","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized)]","latency_us":185628,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:44.121421076Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"10.618922343s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0877265-abb8-4724-973b-3d5714310a6b\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:35:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f0877265-abb8-4724-973b-3d5714310a6b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:46.296798461Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:46.297572559Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"77d08283-c539-4772-9574-63bb9af07711","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39105","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":731,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.712302296Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.712688225Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.712700205Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.733667736Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.733692957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.75090005Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.75092204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.768185894Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.768217705Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.786937984Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.786964034Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.804626578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.804669319Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.822254319Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.822280891Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.840097658Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.840120639Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.857587978Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.857616188Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.857947846Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.858068239Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:54.891438671Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:54.891486532Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:54.891494942Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:54.891501742Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:54.891510232Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:35:54.891618285Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"615f534f-4017-4f68-b565-479d0c27c7d2","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized)]","latency_us":179674,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:54.891826949Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"21.177105438s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"615f534f-4017-4f68-b565-479d0c27c7d2\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:35:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"615f534f-4017-4f68-b565-479d0c27c7d2\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:35:59.143388768Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"7.360949378s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.012346159Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.012551854Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.012558734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.034766314Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.034785615Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.054746082Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.054772713Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.072543289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.07256636Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.091127694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.091153605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.109373771Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.109396642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.126857431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.126892941Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.145277507Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.145293463Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.163577051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.163605781Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.163913468Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.164030531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:16.195513228Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:16.195543549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:16.195551579Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:16.19556172Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:16.19557016Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.195693103Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"091d79f8-ecd6-41f6-95c6-ef63081634e2","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized)]","latency_us":183752,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:16.195889317Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"21.724523078s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"091d79f8-ecd6-41f6-95c6-ef63081634e2\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:36:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"091d79f8-ecd6-41f6-95c6-ef63081634e2\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.250652509Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:16.250756811Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4df9d096-350b-4b0e-9400-cadd8f29d6df","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45475","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":53,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:21.478476125Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"23.70618451s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.862620684Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.862864099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.862876339Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.88509499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.885146441Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.902752053Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.902786134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.920094059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.92013268Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.937861695Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.937900446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.95599838Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.956036611Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.972816274Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.972845214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.989149306Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:37.989186867Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:38.008047239Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:38.0081083Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:38.008384987Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:38.008495229Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:38.040462808Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:38.040483528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:38.040488768Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:38.040493929Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:38.040499029Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:38.040632472Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bbd81dce-6d71-4d91-9ab4-76778af06e2b","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\", TimeNotSynchronized), (\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\", TimeNotSynchronized), (\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\", TimeNotSynchronized), (\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\", TimeNotSynchronized), (\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\", TimeNotSynchronized)]","latency_us":178453,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:36:38.040801266Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"34.873448755s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bbd81dce-6d71-4d91-9ab4-76778af06e2b\", \"content-length\": \"124\", \"date\": \"Fri, 26 Jul 2024 02:36:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bbd81dce-6d71-4d91-9ab4-76778af06e2b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:46.202103679Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49178"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:36:46.202177651Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c508f4dc-39ee-4b67-8f86-ea7354634e48","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49178","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":34,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:00.115819306Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"16.3372196s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.856587757Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.857617991Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.857624841Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.879171104Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.879202585Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.897007511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.897032812Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.914703944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.914740265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.932686114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.932715335Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.951692998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.951721449Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.969693098Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.969721449Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.987630117Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:12.987657568Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.006330074Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.006358085Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.006634131Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.006761624Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.089926606Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.089961507Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.089966227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.089971477Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.089976677Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.174320626Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.178486583Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.183184903Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.188306153Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.192861439Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.197254782Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.202111955Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.206953948Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.211451363Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:13.215796675Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.073099028Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.073218411Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.12291425Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.123066934Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.183375022Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.183488524Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.200239005Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34438"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.200376759Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"fc4af6ff-fe59-45bd-a386-2a1657404ed7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34438","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":63,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 28.348864ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:16.226923358Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.233684146Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.233801329Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.253614014Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:16.253836597Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 25.680438ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:16.26680429Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 124.765604ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:16.27519875Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 86.072486ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:16.319031999Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 41.289359ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:16.490942172Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 269.724923ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:16.802219842Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 122.816043ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:16.9197988Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 131.541212ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.037640727Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 508.204618ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.179903669Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 308.71171ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.325074069Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 48.130816ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.386862342Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 203.586688ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.446383442Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 51.187971ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.557526317Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 31.501421ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.627964332Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 560.721637ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.723528004Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 185.490796ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.78246851Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 81.840943ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.814290753Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 326.194868ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.846830493Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 818.913891ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.870959846Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 298.190969ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.916894749Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 338.21753ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:17.987686632Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 1.168605502s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:18.194526082Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 582.526718ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:18.238971029Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 823.491703ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:18.302404851Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 1.061350432s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:18.347393231Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 1.250754661s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:18.885766524Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 683.207033ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:18.897466767Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.403752628s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:19.140332357Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 608.325449ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:19.374053205Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 721.549449ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:19.418408351Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 782.30112ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:19.59008562Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 520.598787ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:19.992980488Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 1.133632244s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:20.14639581Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 1.034531349s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:20.153474645Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 1.143359559s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:20.382580425Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 708.434097ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:20.52368218Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.190795902s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:20.553062336Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 1.233366511s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:21.197524255Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 764.993467ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:21.240407266Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 887.964392ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:21.28851505Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 938.388542ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:21.534559255Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.168093895s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:21.753442057Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 974.078767ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:22.020059213Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 595.894622ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:22.187351288Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 797.833994ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:22.441284728Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 788.85729ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:22.481814434Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 1.384337995s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:22.793250966Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 820.49341ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:22.932594279Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 907.690129ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:23.005194825Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 1.495564649s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:23.24825774Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 1.081145555s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:23.281298902Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 566.086741ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:23.761998296Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 929.492741ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:23.922224507Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 1.401143149s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:24.187658714Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 725.650675ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:24.336669844Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 942.726238ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:24.371466716Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 1.473989812s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:24.753067216Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 1.453465682s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:24.861160386Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.055013492s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:25.072747271Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 994.79164ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:25.324033227Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 1.106531857s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:25.598861255Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 882.509841ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:26.138320528Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 766.546406ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:26.236771409Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c\") failed: Property not found. retry in 1.343264458s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:26.323940984Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 591.839421ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:26.331544602Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63\") failed: Property not found. retry in 1.035664564s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:26.714660235Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8\") failed: Property not found. retry in 883.763775ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:26.933784974Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3\") failed: Property not found. retry in 1.132701167s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:27.01328854Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 597.642464ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:27.029765825Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 509.598542ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:27.637412194Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 802.874275ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:28.168819902Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.219860824s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:28.981147872Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.374930606s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:30.214211305Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 576.195772ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:31.600276051Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.255048057s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:32.187645576Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:32.550356996Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"55.643777239s"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.486172572s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:33.456050925Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 801.922264ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:34.953352999Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.221089642s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:35.766031966Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.038808169s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:36.998006184Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 877.948863ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:38.048142996Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 660.291266ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:38.936250704Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 723.022259ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:39.606432433Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.086410964s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:40.338536629Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.202212782s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:41.434998403Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 858.971062ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:42.646745939Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.190622235s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:43.514931031Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6\") failed: Property not found. retry in 1.488953671s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:37:44.717654706Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.2282941Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_cockroachdb_9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.228361111Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_cockroachdb_dd312e93-ad11-47cc-8ead-ef2eaf27f3a3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.228368181Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_cockroachdb_31365180-26ce-40d1-a6d6-dd6b5fa83fe8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.228373932Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_cockroachdb_3f64979c-8afb-403e-8824-2c7f0af65e0c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.228380992Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_cockroachdb_5f42bb99-520b-4711-ae44-83f79f9b2f63"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.228408912Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.234828172Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37805"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.235123239Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f4551aa7-6ddd-4cde-b253-4dd3db5a6481","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37805","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":238,"response_code":"200"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.274500498Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.374643107Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a97c8d42-c790-4e03-be72-5aeb0a466fbd","remote_addr":"[fd00:1122:3344:101::1]:52946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":33516914,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.413538125Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:51705"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:46.441669122Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3481"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:50.306198621Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3499"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.631016373Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3522"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.631146866Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"592045c1-22c3-450d-900e-9423a2ef8cb9","remote_addr":"[fd00:1122:3344:101::1]:51705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":13218177,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 25724e7e-f0f3-413d-ab41-b4a33973050b, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 82379658-84e2-487b-bcd4-b30ffc1d9874, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0eb4388e-1e4e-49da-b41b-58746266c965, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bbb96e05-c4c2-4d74-93ab-32168e25b9fd, kind: Service { id: 0eb4388e-1e4e-49da-b41b-58746266c965 }, name: Name(\"ntp-0eb4388e-1e4e-49da-b41b-58746266c965\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 143, 87])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }, OmicronZoneConfig { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 2f46bf86-4fc6-46a8-b573-bf2fb626f25d, kind: Service { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425 }, name: Name(\"external-dns-cf7587f8-33d7-4e89-bad7-ae8cdd556425\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 111])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 53979846-b429-4bae-9968-d96ff860d07e, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 567092fa-fb08-4940-8635-631e5c908557, kind: Service { id: 53979846-b429-4bae-9968-d96ff860d07e }, name: Name(\"external-dns-53979846-b429-4bae-9968-d96ff860d07e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 230])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b4031b8-df71-45f8-b977-b6aa00214087, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 824ae30e-19ba-423f-923a-3afa2a59b03f, kind: Service { id: 8b4031b8-df71-45f8-b977-b6aa00214087 }, name: Name(\"nexus-8b4031b8-df71-45f8-b977-b6aa00214087\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 145])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 56f1d4b9-214d-4104-bc80-a770b9be089e, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: e94e36b7-3a4e-4066-8a04-518f25a7ffdb, kind: Service { id: 56f1d4b9-214d-4104-bc80-a770b9be089e }, name: Name(\"nexus-56f1d4b9-214d-4104-bc80-a770b9be089e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 23])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 95c06611-4118-4e93-84ac-1b68f8ff2a6d, kind: Service { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087 }, name: Name(\"nexus-9d80bc64-2ec6-4d4e-96d7-5c012d075087\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8752c2d7-36d8-49ed-860d-ec196a32b72c, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: d6874d80-d20e-408e-91a2-c5a467770626, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 78641540-4174-48c6-892e-29962a5829c6, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 2cc93cb4-8546-4d34-af83-babf2273221c, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 290da25c-6635-4875-bb74-2c418e3e33a5, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0a17296b-a160-4ef4-9cb7-01d58322092f, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2fa8c446-5b0d-456d-a750-28a177c9ab27, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External } } } }, OmicronZoneConfig { id: b56f7823-5e5f-4958-a8ee-f08cb6a361c1, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External } } } }, OmicronZoneConfig { id: 07eec7c5-c6b0-4973-a922-2fdc5f972621, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External } } } }, OmicronZoneConfig { id: 39533c6a-2c55-4972-8616-1a19033efc8c, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2fc14079-6244-4be0-84a1-1c701868ee2d, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7832441b-d9a6-4b73-a71f-8fb8e621ab9f, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 636d501d-962a-441b-8898-3876ad1cb8b3, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.666631585Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.666788069Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:62928"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.667580217Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.667590117Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 25724e7e-f0f3-413d-ab41-b4a33973050b, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.690714257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.690770769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 4e64b0cb-e3e2-4959-adf4-d5cd70f25b40, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.713335076Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.713379386Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 82379658-84e2-487b-bcd4-b30ffc1d9874, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.734479769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.73452347Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 31365180-26ce-40d1-a6d6-dd6b5fa83fe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.75336102Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.753398581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3f64979c-8afb-403e-8824-2c7f0af65e0c, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.774062734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.774104135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: dd312e93-ad11-47cc-8ead-ef2eaf27f3a3, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.794295826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.794335897Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 9374a3cd-ae44-4f04-9fc8-6112fb6fc7f6, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.814270442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.814311774Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5f42bb99-520b-4711-ae44-83f79f9b2f63, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cf7587f8-33d7-4e89-bad7-ae8cdd556425, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.83430765Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.834358002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: cf7587f8-33d7-4e89-bad7-ae8cdd556425, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 53979846-b429-4bae-9968-d96ff860d07e, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.909278091Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.909331982Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 53979846-b429-4bae-9968-d96ff860d07e, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6874d80-d20e-408e-91a2-c5a467770626, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.985373798Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:37:59.985426309Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d6874d80-d20e-408e-91a2-c5a467770626, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0a17296b-a160-4ef4-9cb7-01d58322092f, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.067144838Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.067198869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0a17296b-a160-4ef4-9cb7-01d58322092f, dataset_name: DatasetName { pool_name: ZpoolName { id: 4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2fa8c446-5b0d-456d-a750-28a177c9ab27, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.146635463Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.146687544Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 2fa8c446-5b0d-456d-a750-28a177c9ab27, dataset_name: DatasetName { pool_name: ZpoolName { id: b8ef9471-5b81-40ed-90b5-1103a4a9eb91 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.222087115Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.222134446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23, dataset_name: DatasetName { pool_name: ZpoolName { id: a47af5d0-e565-4513-84ad-c553b7265a72 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b56f7823-5e5f-4958-a8ee-f08cb6a361c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.290736888Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.290788159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b56f7823-5e5f-4958-a8ee-f08cb6a361c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a33f7b1-2bfd-4239-9138-fb534f4ccb3f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 07eec7c5-c6b0-4973-a922-2fdc5f972621, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.376998132Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.377066984Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 07eec7c5-c6b0-4973-a922-2fdc5f972621, dataset_name: DatasetName { pool_name: ZpoolName { id: a38a1ea7-44a1-411f-bbb8-767c9a23135f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39533c6a-2c55-4972-8616-1a19033efc8c, dataset_name: DatasetName { pool_name: ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.448616955Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.448675336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 39533c6a-2c55-4972-8616-1a19033efc8c, dataset_name: DatasetName { pool_name: ZpoolName { id: cb102b97-0352-4c03-8d37-4c80c4475510 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2fc14079-6244-4be0-84a1-1c701868ee2d, dataset_name: DatasetName { pool_name: ZpoolName { id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.518667161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.518711782Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 2fc14079-6244-4be0-84a1-1c701868ee2d, dataset_name: DatasetName { pool_name: ZpoolName { id: c5566b16-9a9f-498b-a4b1-83ac43308db5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7832441b-d9a6-4b73-a71f-8fb8e621ab9f, dataset_name: DatasetName { pool_name: ZpoolName { id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.583933265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.583985716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 7832441b-d9a6-4b73-a71f-8fb8e621ab9f, dataset_name: DatasetName { pool_name: ZpoolName { id: 136ed548-ab63-44da-a187-0fa13dc111a0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 636d501d-962a-441b-8898-3876ad1cb8b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.649801083Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.649856484Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 636d501d-962a-441b-8898-3876ad1cb8b3, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.717248228Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.717316109Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.717591386Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.71777927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.840367543Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.840592858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.855287831Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.855507116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.855540137Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.898531721Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.898589502Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.898594042Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.898597803Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:00.898601573Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.073160139Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.075217517Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.139807435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.14003468Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.140099141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.140163663Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.140170743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.140217884Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.140230335Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.140298606Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.145644861Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.150649718Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.155611034Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.160913707Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.165986656Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.171046874Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.176170274Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.30192189Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.306994659Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.311782111Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.31644545Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.321363674Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.326249379Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.331302717Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.336245372Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.341349231Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.347527615Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.354485748Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.360913738Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.366091059Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.372473228Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.380033164Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.385866531Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.392144167Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.398625258Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.405159301Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.412450972Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.475011832Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.55583798Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.734826509Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.831961548Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.90784613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.910163814Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.910179564Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.976249027Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 168, 111]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.976279058Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.979484683Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:01.980034786Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.039348511Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 180, 23]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.039395442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.043008186Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.043027897Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.103695973Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 158, 230]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.103727864Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.104955973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.104984273Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.162601559Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 248, 148]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.16263688Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.164345519Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.16435958Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.221992106Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 161, 145]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.222028977Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.222971119Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.222989089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.225211361Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.225223791Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.225363274Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.225379085Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.225439066Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.225457946Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.225513008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.225524808Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.228508498Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.23032856Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.230450673Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.303861788Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.304256027Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.308939446Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.312188452Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.312571261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.312608742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.573991456Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:02.713337579Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.277635396Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.284909566Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.29407032Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.303301125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.31292427Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.321309056Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.331664808Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.346544515Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:03.75012042Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.295578991Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.299113564Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.357068867Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.358708505Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.359861243Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Booting oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.371092425Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.415296317Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.417773555Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.418049061Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.431375533Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.435604831Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.435910498Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.450506159Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.454568434Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.45480592Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Booting oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.463949353Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.474156592Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.477430218Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.477639903Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Booting oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.486830848Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.497110417Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.49722452Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Booting oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.507141262Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Booting oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.516767487Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.757361145Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:08.757622411Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 62.769952ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.227012082Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 38.064834ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.242832122Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 31.321397ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.256966792Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 60.245863ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.267787014Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 66.723357ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.278818372Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 60.038278ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.28986155Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 31.118792ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.300521709Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 63.032108ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.340503602Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 31.240491ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.389413294Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 73.670778ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.458328914Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 66.215613ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.500517729Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 134.263247ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.529201479Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 63.518001ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.555934413Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 132.828986ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.576250657Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 111.871589ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.595026316Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 68.121458ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.621254718Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 101.904253ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:09.763039199Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 139.762496ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.127891848Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 79.274963ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.199564062Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 42.369869ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.237977889Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 49.49862ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.368803904Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 142.712361ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.661946409Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 149.044898ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.707993815Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 70.931729ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.747529598Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 241.077888ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.764673298Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 133.927827ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.79257725Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 215.898263ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.819070519Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 142.510647ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.844525753Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 144.125509ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.86494369Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 233.852339ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.880959624Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 124.786925ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.895011842Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 250.587946ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.909673805Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 138.698342ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.922674168Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 114.572304ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.938489827Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 299.928998ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.951640665Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 154.987216ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.963272806Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 250.667616ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.978403899Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 251.141395ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:10.989417867Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 206.525984ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.001006347Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 270.652792ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.021498796Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 385.483751ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.034823547Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 372.7807ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.049614282Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 234.297889ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.063600309Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 235.876182ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.075759463Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 272.239825ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.088578282Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 492.033581ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.130992242Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 419.893031ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.145729796Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 333.90613ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.177976749Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 801.691172ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.221614978Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 393.881748ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.239456135Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 946.235077ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.250847381Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 360.712582ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.266165698Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 635.97115ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.305928797Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 438.452703ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.335081028Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 280.41401ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.347865886Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 400.079574ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.374326304Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 546.751043ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.435128114Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 539.115136ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.448121407Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 975.690754ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.524588483Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 726.845574ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.582647119Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 719.841747ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.637010038Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 925.243284ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.647400531Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.150291448s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.6614729Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.077458443s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.678042557Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:11.726065348Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:11.726347945Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 908.500712ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.789682244Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 502.955627ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.803983048Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 883.713327ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.959664983Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 61.438358ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.980664483Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 769.272173ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:11.994789343Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 528.668904ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.015603489Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 549.953662ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.043703335Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 76.026082ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.05545181Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 271.208066ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.143997837Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.299869702s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.214056723Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 757.378214ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.322987536Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 1.207117875s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.337107786Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 939.982446ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.376055026Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 565.363568ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.429576876Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 500.044382ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.513467064Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.051649901s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.558570078Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 998.529038ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.591927217Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 647.652897ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.60746626Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 1.175217273s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.71112735Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.347277943s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.770583789Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 540.030386ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.782702272Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 549.205695ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.825002939Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 1.051217282s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:12.856784552Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 591.971869ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.007788658Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 598.551122ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.025495721Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 694.714218ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.095777132Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 728.170445ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.266614201Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 531.813303ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.32696477Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 686.609232ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.336207026Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.212999918s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.383503271Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.053060177s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.525283271Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 638.215234ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.556648954Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 692.036866ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.601255125Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 1.157625353s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.6695252Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 703.566401ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.681810697Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 1.020846119s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.692918626Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 559.881043ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.80442578Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 759.721109ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.872086Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 616.541054ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.898733982Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 1.089549693s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:13.9200657Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 1.418846584s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.007475402Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 658.991775ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.037555494Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.228968742s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.130391101Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 789.52309ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.25924459Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 679.592392ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.305246604Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 772.788959ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.377687986Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.252796076s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.398257006Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:14.44808822Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:14.44811218Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:14.448282855Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:14.448456939Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Booting oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:14.455286018Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Booting oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:14.461143975Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 916.547566ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.529335907Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.223240924s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.591689803Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.314174838s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.610424041Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 516.190027ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.644272601Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 55.289096ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.66775336Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 47.534541ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.677840915Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 54.173804ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.690789767Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 911.950162ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.709944265Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 1.015841794s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.725682062Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 77.135721ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.734672972Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 125.308535ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.743779205Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 83.188287ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.755340975Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 199.491411ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.82280733Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 700.662958ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.843322079Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 268.100432ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.857265545Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 283.666092ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.886092928Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 617.715149ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:14.998484053Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 1.333748753s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.022724819Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 571.29594ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.03475949Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 571.491964ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.062572649Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 206.764445ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.139749161Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 973.037101ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.162029081Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 1.307068184s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.173350145Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 599.598136ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.183319718Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 441.910557ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.529131494Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 1.495780293s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.547537283Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 700.10068ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.563976017Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 502.666207ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.581832464Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.089230904s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.636245655Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 1.134026384s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.654276376Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.191226292s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.670667919Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 1.092963024s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.687248136Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 750.467757ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.701233142Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 759.175374ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.71440973Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 1.26755065s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.755975841Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 1.10489113s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.797030899Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.22240609s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.828454133Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.202442041s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.938939413Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 980.779676ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:15.986148846Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 723.669448ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.102157155Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:16.134618043Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:16.13495441Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 858.165153ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.157841355Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 1.468255589s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.278775169Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 911.433337ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.372453927Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 54.974384ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.402960759Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:16.421321908Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:16.4282811Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6b11bdfb-2909-42bc-95f1-1658b271dffb","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61217","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6025,"response_code":"200"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 1.196137983s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.465765675Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 110.966544ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.474856938Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 1.038957119s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.486335406Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 596.922838ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.503286542Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 249.449886ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.610798662Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 885.682331ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.738532605Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 635.811009ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.792664409Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 1.316509253s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.805797156Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 780.095482ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.847816947Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 429.419806ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.900711812Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.484845151s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.922659095Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 1.4148461s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.93787915Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 1.164696839s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:16.980052345Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 1.466949806s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.030080643Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 948.950213ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.043802234Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 1.420350353s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.063418132Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 503.780855ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.081541275Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 1.038997676s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.117398722Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 690.450748ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.155508272Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 1.28035267s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.294830515Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 984.582926ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.340351958Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 1.145139534s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.439825341Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 746.331141ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.537056021Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.359587617s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.595311852Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 735.26922ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.634187119Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 758.025116ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.643308972Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 842.246859ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.67103063Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 1.060694638s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.756940846Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 709.937301ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:17.85644347Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 859.413568ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.003250688Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 535.783487ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.132734681Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 1.196678349s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.156109126Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 664.495066ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.179447742Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 889.919098ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.294099519Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 644.780072ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.335297901Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 896.317074ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.361992244Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 853.835298ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.3793596Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 775.515228ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.412701278Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 607.460536ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.423077621Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 993.421556ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.493583117Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 844.824562ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.505391093Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 835.775124ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.522845061Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 807.663226ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.576123835Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 892.108672ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.583694091Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 913.859375ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.59349636Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 1.376684481s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.677897471Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 965.107967ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.831751544Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 1.249514832s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.858342845Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 725.377256ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.875801233Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.286571892s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.967725259Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 1.422678562s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:18.9925762Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 840.411529ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.04312416Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 697.893672ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.199157613Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 1.196194513s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.211391999Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.052716839s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.247035181Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 532.236256ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.271641226Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:19.335270301Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:19.335541188Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 1.459445731s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.364966875Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 1.305111251s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.377238832Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 719.557575ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.388577606Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.138770345s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.406847823Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 935.804181ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.487811614Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 1.39537767s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.500455739Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 1.44383283s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.524871459Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 37.740104ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.562135909Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 111.375532ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.610207162Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 1.313422206s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.619727374Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 242.576827ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.819773175Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 1.428264671s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.853585485Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 1.262536331s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.867768516Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 646.050198ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.897678515Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 1.250749465s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:19.911025786Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 653.58917ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.06535235Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 594.263395ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.074180056Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 986.617776ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.117429156Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 981.426939ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.126077758Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.393869111s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.267147532Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 826.351109ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.30946403Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 667.929668ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.417914773Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 557.359341ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.426502183Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 867.19502ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.436281181Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.092237915s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.554786189Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.299264507s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.565684833Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.083926469s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.682150703Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 770.086006ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.691641545Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 1.390809443s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.732155771Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 696.565888ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.842132039Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 1.346052844s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.91027241Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 554.072078ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.943095796Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 1.06482612s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:20.984636987Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 855.830191ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.00148653Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 781.893403ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.099682423Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 1.341537553s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.118362259Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 750.945395ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.132469679Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 950.712549ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.144999111Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.087172881s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.156004418Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 608.581336ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.17278706Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 797.983157ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.294566594Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 735.889379ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.316943946Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 1.074853337s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.474014934Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 1.080246636s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.510310922Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 1.094737257s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.551155166Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 507.402913ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.658665406Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.131050331s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.673451221Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.369405875s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.778809132Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 873.157157ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.799016514Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 612.710816ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.8686653Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.085287475s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.880448255Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 1.131818642s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.89818466Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 585.06214ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:21.908969531Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 1.417752912s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.073060633Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 571.733707ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.091223417Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 1.057008939s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.116843545Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 581.239017ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.143500057Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 1.287894678s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.166551616Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.061535996s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.1868708Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 860.785529ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.265423624Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 1.063532381s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.289454056Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 630.91332ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.474788513Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 1.24689339s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.500791921Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 761.627155ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.516713403Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 1.448967794s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.566344642Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 928.618566ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.608133467Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 1.398402249s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.665231451Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 970.858464ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.676467093Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 552.72897ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.691974815Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 1.246253448s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.742330541Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 710.79563ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.820604399Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.330003901s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:22.980857891Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 776.756091ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.04589772Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 1.255162178s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.121960606Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 631.217895ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.140568591Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 929.111352ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.162477332Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 1.051856683s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.194878139Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 785.469626ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.259428656Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.239008194s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.273145396Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 540.229297ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.293541923Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 1.229397918s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.368016302Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 1.125697455s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.471469108Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 1.148853328s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.505911032Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 1.05879238s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.546823578Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 1.448035711s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.559460013Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 563.735643ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.663932252Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 933.78244ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.766406975Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.372499075s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.787428806Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 1.074200196s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.83812107Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 1.30426028s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:23.850421137Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 1.103708218s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.004123636Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 1.183857099s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.030307138Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 657.82356ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.0586926Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 956.872623ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.080703894Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.363408598s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.107255184Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 888.233221ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.244256923Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 602.795563ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.263973664Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 571.683818ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.32162858Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 1.257850719s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.390049937Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 623.709386ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.523517864Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 1.092516188s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.60940443Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 971.714061ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.618932142Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6\") failed: Property not found. retry in 908.962739ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.62826938Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 1.232566307s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.66635968Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 1.470427323s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.712431936Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 654.284243ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.728758797Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c\") failed: Property not found. retry in 947.114385ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.880161393Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.001180436s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.911612477Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 652.197491ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:24.926317651Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 913.500077ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.019455425Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 817.553331ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.048734169Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 831.412638ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.122506341Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 730.890884ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.144350901Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.095517108s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.159076115Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 873.484632ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.168222399Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.016962993s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.180482515Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 1.289672256s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.225525297Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 988.581379ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.395782242Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 838.881866ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.484781661Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 918.820228ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.602868588Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 961.825284ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.613251421Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 888.882175ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.657525165Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 755.076887ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.711240919Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 638.341542ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.876424836Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3\") failed: Property not found. retry in 1.377837355s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.885747674Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 614.207066ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.912579881Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1\") failed: Property not found. retry in 1.338791258s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.922296088Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5\") failed: Property not found. retry in 954.453482ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.944117637Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 635.620711ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:25.962608469Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087\") failed: Property not found. retry in 1.071010764s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.05259919Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 933.621817ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.194841941Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d\") failed: Property not found. retry in 1.249449624s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.208415348Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27\") failed: Property not found. retry in 1.45493989s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.265244745Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 680.951464ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.334171585Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c\") failed: Property not found. retry in 1.453915398s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.395401505Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f\") failed: Property not found. retry in 1.180273814s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.477335018Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f\") failed: Property not found. retry in 1.157508516s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.526815044Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e\") failed: Property not found. retry in 866.546369ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.537538644Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087\") failed: Property not found. retry in 1.478837272s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.549835511Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23\") failed: Property not found. retry in 1.199145207s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.563887009Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c\") failed: Property not found. retry in 1.075607011s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.575288316Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621\") failed: Property not found. retry in 1.343455851s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.590441569Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e\") failed: Property not found. retry in 986.387792ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:26.611203234Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 824.604189ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:27.02597205Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425\") failed: Property not found. retry in 1.134148103s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:27.149352661Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.373121892s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:27.895434803Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:28.282954582Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"86.666485233s"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 674.464092ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:29.285049192Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.392860821s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:29.974926392Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.136275398s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:31.378411874Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.100014408s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:32.525693434Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.127790409s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:33.636611176Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.277560012s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:34.776638888Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.054270165s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:36.06617458Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.096816305s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:37.131576087Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626\") failed: Property not found. retry in 1.108666427s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:38:38.239890488Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.377993754Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_07eec7c5-c6b0-4973-a922-2fdc5f972621"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378082836Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_cf7587f8-33d7-4e89-bad7-ae8cdd556425"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378100676Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_2cc93cb4-8546-4d34-af83-babf2273221c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378106336Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_7832441b-d9a6-4b73-a71f-8fb8e621ab9f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378111797Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_56f1d4b9-214d-4104-bc80-a770b9be089e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378116777Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_1dcc3b27-f3d4-4d4b-b4c7-1bf76f0e0e23"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378122327Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_2fa8c446-5b0d-456d-a750-28a177c9ab27"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378130077Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_clickhouse_d6874d80-d20e-408e-91a2-c5a467770626"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378138657Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_290da25c-6635-4875-bb74-2c418e3e33a5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378144987Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_oximeter_8752c2d7-36d8-49ed-860d-ec196a32b72c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378150077Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_external_dns_53979846-b429-4bae-9968-d96ff860d07e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378155647Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_9d80bc64-2ec6-4d4e-96d7-5c012d075087"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378163208Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_0a17296b-a160-4ef4-9cb7-01d58322092f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378175418Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_b56f7823-5e5f-4958-a8ee-f08cb6a361c1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378185048Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_39533c6a-2c55-4972-8616-1a19033efc8c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378192478Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_2fc14079-6244-4be0-84a1-1c701868ee2d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.378197428Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_636d501d-962a-441b-8898-3876ad1cb8b3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.38211491Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_crucible_pantry_78641540-4174-48c6-892e-29962a5829c6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.38212584Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3058","zone":"oxz_nexus_8b4031b8-df71-45f8-b977-b6aa00214087"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.382681173Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.38640315Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.391603792Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7b78f765-bc5c-4be0-becf-e806b992c041","remote_addr":"[fd00:1122:3344:101::1]:62928","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":39724532,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.412563061Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1250"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.413890622Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.413986274Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.421571341Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:687"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.428452642Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:700"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.462952398Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:831"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/6101b963-ee1b-430c-ad60-5a5922d708b7/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.469791687Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/6101b963-ee1b-430c-ad60-5a5922d708b7/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.573383346Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/6101b963-ee1b-430c-ad60-5a5922d708b7/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.707416956Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/6101b963-ee1b-430c-ad60-5a5922d708b7/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.837701639Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/6101b963-ee1b-430c-ad60-5a5922d708b7/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:39.939962707Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a378e554-cc84-4c79-8a66-24c0f5848115\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:40 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"6101b963-ee1b-430c-ad60-5a5922d708b7\\\"\", request_id: \"a378e554-cc84-4c79-8a66-24c0f5848115\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:40.440512625Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"949a7fad-f12d-4a6a-96b2-992bfa2d2f28\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:41 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"6101b963-ee1b-430c-ad60-5a5922d708b7\\\"\", request_id: \"949a7fad-f12d-4a6a-96b2-992bfa2d2f28\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:41.05138486Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"532e50af-9a0d-44d1-a632-4d7b3cfb177b\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:40 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"6101b963-ee1b-430c-ad60-5a5922d708b7\\\"\", request_id: \"532e50af-9a0d-44d1-a632-4d7b3cfb177b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:41.276801093Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e9b72a64-9bf4-4dd3-8a12-8e0643fff4e1\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"e9b72a64-9bf4-4dd3-8a12-8e0643fff4e1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:42.651058013Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ea7cb684-b721-41be-91eb-a876df47456b\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:42 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"ea7cb684-b721-41be-91eb-a876df47456b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:43.588878323Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"856a373a-8c32-43a0-a562-17c5cbf9bd6d\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:44 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"856a373a-8c32-43a0-a562-17c5cbf9bd6d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:44.303094131Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c1a32466-fcb6-44b3-ad34-6cd5c448ef27\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:44 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"c1a32466-fcb6-44b3-ad34-6cd5c448ef27\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:45.143368643Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:46.500983575Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51809"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:46.50120128Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"732670bc-1751-438c-8e2b-91ef39adbf80","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51809","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8561b594-00db-4d2e-b8c5-fe3ba07f4584\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:46 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"8561b594-00db-4d2e-b8c5-fe3ba07f4584\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:46.521292719Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe0fc5ad-8be3-4275-b3e6-b9b35bdc21b7\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:47 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"fe0fc5ad-8be3-4275-b3e6-b9b35bdc21b7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:47.458530175Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"251cae66-8379-40fe-a0b6-e870e447febc\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:48 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"251cae66-8379-40fe-a0b6-e870e447febc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:48.519814847Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c8c9201c-8a37-4f6b-ba31-e2a970716923\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:49 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"c8c9201c-8a37-4f6b-ba31-e2a970716923\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:49.774598848Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"40a6c0e3-ae01-48a2-a95a-aabc3fc132ff\", \"content-length\": \"173\", \"date\": \"Fri, 26 Jul 2024 02:38:52 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"ea0e0fc4-fc93-4de6-b687-98567d824e83\\\"\", request_id: \"40a6c0e3-ae01-48a2-a95a-aabc3fc132ff\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:52.211188385Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:52.821228221Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"sled-agent/src/nexus.rs:573","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:52.913449674Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33771"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:52.915247036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:53.009562069Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"3dc74787-9f50-40d0-8bf5-6aac9018dec5","remote_addr":"[fd00:1122:3344:101::a]:33771","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95285,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:53.010110572Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:53.010182673Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:53.015639711Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:53.017164486Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"d62ba17a-e048-4b10-9133-11c68fb5a0ea","remote_addr":"[fd00:1122:3344:101::a]:33771","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.643003594Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63957"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.646985687Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2f46bf86-4fc6-46a8-b573-bf2fb626f25d, kind: Service { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425 }, name: Name(\"external-dns-cf7587f8-33d7-4e89-bad7-ae8cdd556425\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 111])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 567092fa-fb08-4940-8635-631e5c908557, kind: Service { id: 53979846-b429-4bae-9968-d96ff860d07e }, name: Name(\"external-dns-53979846-b429-4bae-9968-d96ff860d07e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 230])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 824ae30e-19ba-423f-923a-3afa2a59b03f, kind: Service { id: 8b4031b8-df71-45f8-b977-b6aa00214087 }, name: Name(\"nexus-8b4031b8-df71-45f8-b977-b6aa00214087\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 145])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95c06611-4118-4e93-84ac-1b68f8ff2a6d, kind: Service { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087 }, name: Name(\"nexus-9d80bc64-2ec6-4d4e-96d7-5c012d075087\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e94e36b7-3a4e-4066-8a04-518f25a7ffdb, kind: Service { id: 56f1d4b9-214d-4104-bc80-a770b9be089e }, name: Name(\"nexus-56f1d4b9-214d-4104-bc80-a770b9be089e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 23])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.647193062Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.647242103Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.647257493Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.647272704Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.647288694Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.647314115Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.647857607Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d831668c-f727-47ab-9649-28141ecabd55","remote_addr":"[fd00:1122:3344:101::b]:63957","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":4293,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.65012501Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:903"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.878210986Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52248"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.878322249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.9742645Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"4bc8082d-ee93-4d6c-8cfe-df990454b809","remote_addr":"[fd00:1122:3344:101::b]:52248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95953,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.974718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.974766311Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.976145293Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:55.97641237Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"6f20aced-4af3-45ed-a808-fbf657e451f1","remote_addr":"[fd00:1122:3344:101::b]:52248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1759,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.236791039Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55604"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.237020735Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2f46bf86-4fc6-46a8-b573-bf2fb626f25d, kind: Service { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425 }, name: Name(\"external-dns-cf7587f8-33d7-4e89-bad7-ae8cdd556425\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 111])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 567092fa-fb08-4940-8635-631e5c908557, kind: Service { id: 53979846-b429-4bae-9968-d96ff860d07e }, name: Name(\"external-dns-53979846-b429-4bae-9968-d96ff860d07e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 230])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 824ae30e-19ba-423f-923a-3afa2a59b03f, kind: Service { id: 8b4031b8-df71-45f8-b977-b6aa00214087 }, name: Name(\"nexus-8b4031b8-df71-45f8-b977-b6aa00214087\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 145])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95c06611-4118-4e93-84ac-1b68f8ff2a6d, kind: Service { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087 }, name: Name(\"nexus-9d80bc64-2ec6-4d4e-96d7-5c012d075087\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e94e36b7-3a4e-4066-8a04-518f25a7ffdb, kind: Service { id: 56f1d4b9-214d-4104-bc80-a770b9be089e }, name: Name(\"nexus-56f1d4b9-214d-4104-bc80-a770b9be089e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 23])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.237084236Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.237116657Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.237131137Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.237136917Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.237143898Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.237155578Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.237177718Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"570aafb1-dac0-4d97-9e7a-97e00b5edacb","remote_addr":"[fd00:1122:3344:101::b]:55604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":348,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.304862009Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46550"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.305738979Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2f46bf86-4fc6-46a8-b573-bf2fb626f25d, kind: Service { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425 }, name: Name(\"external-dns-cf7587f8-33d7-4e89-bad7-ae8cdd556425\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 111])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 567092fa-fb08-4940-8635-631e5c908557, kind: Service { id: 53979846-b429-4bae-9968-d96ff860d07e }, name: Name(\"external-dns-53979846-b429-4bae-9968-d96ff860d07e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 230])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 824ae30e-19ba-423f-923a-3afa2a59b03f, kind: Service { id: 8b4031b8-df71-45f8-b977-b6aa00214087 }, name: Name(\"nexus-8b4031b8-df71-45f8-b977-b6aa00214087\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 145])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95c06611-4118-4e93-84ac-1b68f8ff2a6d, kind: Service { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087 }, name: Name(\"nexus-9d80bc64-2ec6-4d4e-96d7-5c012d075087\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e94e36b7-3a4e-4066-8a04-518f25a7ffdb, kind: Service { id: 56f1d4b9-214d-4104-bc80-a770b9be089e }, name: Name(\"nexus-56f1d4b9-214d-4104-bc80-a770b9be089e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 23])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.305798651Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.305829391Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.305836772Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.305843142Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.305859832Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.305865232Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.305877123Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"842237a5-c872-4f06-975e-a93d62a0aae5","remote_addr":"[fd00:1122:3344:101::c]:46550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":983,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.323783821Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49705"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.32459522Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2f46bf86-4fc6-46a8-b573-bf2fb626f25d, kind: Service { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425 }, name: Name(\"external-dns-cf7587f8-33d7-4e89-bad7-ae8cdd556425\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 111])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 567092fa-fb08-4940-8635-631e5c908557, kind: Service { id: 53979846-b429-4bae-9968-d96ff860d07e }, name: Name(\"external-dns-53979846-b429-4bae-9968-d96ff860d07e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 230])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 824ae30e-19ba-423f-923a-3afa2a59b03f, kind: Service { id: 8b4031b8-df71-45f8-b977-b6aa00214087 }, name: Name(\"nexus-8b4031b8-df71-45f8-b977-b6aa00214087\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 145])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95c06611-4118-4e93-84ac-1b68f8ff2a6d, kind: Service { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087 }, name: Name(\"nexus-9d80bc64-2ec6-4d4e-96d7-5c012d075087\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e94e36b7-3a4e-4066-8a04-518f25a7ffdb, kind: Service { id: 56f1d4b9-214d-4104-bc80-a770b9be089e }, name: Name(\"nexus-56f1d4b9-214d-4104-bc80-a770b9be089e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 23])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.324636591Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.324675071Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.324696842Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.324714392Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.324734893Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.324744823Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:38:56.324761193Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1e0d9487-91b9-4538-a67c-bab76ebd8870","remote_addr":"[fd00:1122:3344:101::a]:49705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":879,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:02.55384528Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35969"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:02.554566696Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2e5516a2-f204-468c-91a4-10bdaf0643ef","remote_addr":"[fd00:1122:3344:101::b]:35969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":690,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:02.621048039Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d3a00726-7e8b-4421-aa67-a9c5b4ba774b","remote_addr":"[fd00:1122:3344:101::b]:35969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1668,"response_code":"204"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:09.765158183Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:10.914756247Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53905"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:10.914853989Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"076c3444-381b-4c9f-8e7e-0d7f79709102","remote_addr":"[fd00:1122:3344:101::b]:53905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:10.922113069Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"84c3c589-cd20-4763-a187-f3e26c2d0bf2","remote_addr":"[fd00:1122:3344:101::b]:53905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.046677918Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40573"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.046833511Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"1918c42e-0d2d-437e-b572-6266caaa40e7","remote_addr":"[fd00:1122:3344:101::c]:40573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.084275536Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50293"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.084323907Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"4d5cc864-c371-42b1-9223-721dc065001b","remote_addr":"[fd00:1122:3344:101::c]:50293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":25,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.101288122Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"41c6436c-3e28-4837-a474-39d28a84c375","remote_addr":"[fd00:1122:3344:101::c]:50293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.147290646Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44853"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.147947932Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"8575098f-1e6b-4509-bf17-b17008ff801a","remote_addr":"[fd00:1122:3344:101::b]:44853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":626,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.26476497Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59495"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.265478476Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.329907811Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.353653065Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"4e60e1f4-3252-4f9b-9e20-0c99e1566d4d","remote_addr":"[fd00:1122:3344:101::b]:59495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88848,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.353769948Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52201"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.35386243Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.375489035Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.400026038Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f70b908c-f6c1-4003-b616-021a2780dd91","remote_addr":"[fd00:1122:3344:101::c]:52201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46200,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.402643449Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"57249559-05d9-4d09-9192-229bc30d963b","remote_addr":"[fd00:1122:3344:101::c]:52201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2288,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.40267889Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"61ce9c2f-58fe-4ef5-ba3e-83adc904b8a5","remote_addr":"[fd00:1122:3344:101::b]:59495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2380,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.430381827Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.430445719Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.431996295Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.432719362Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.432731592Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"764314a6-2403-495c-92fb-8263b05c0a7f","remote_addr":"[fd00:1122:3344:101::c]:52201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":4637,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.832228841Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35973"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.832450476Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d84aea72-dc34-4a03-8f0c-1d29aa010dbe","remote_addr":"[fd00:1122:3344:101::a]:35973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.90026738Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.900362062Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"7d756b68-87cd-4e8e-a660-1f76cc3e39f8","remote_addr":"[fd00:1122:3344:101::a]:49242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:11.915338332Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b1185115-0ad7-48e1-8462-b7641e9bd20b","remote_addr":"[fd00:1122:3344:101::a]:49242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":559,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:12.008962048Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:40088"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:12.00903965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:12.033768937Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:12.056282733Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a92a06f3-edc7-4ecd-9494-2fb6efd5c659","remote_addr":"[fd00:1122:3344:101::a]:40088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46747,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:12.058451394Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d744678e-b346-4032-b32e-d9df8258ab33","remote_addr":"[fd00:1122:3344:101::a]:40088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:16.534018335Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52498"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:16.534102097Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"0572b3e0-e0ff-47dc-9e38-ef2008728561","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52498","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:40.968534168Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:40.96861237Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"6fd0790e-caaa-4633-83f7-751a36c5dcff","remote_addr":"[fd00:1122:3344:101::c]:61861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:40.98232554Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37412"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:40.98316911Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"9725d9cc-ce61-43f0-a4d2-2975c2120d01","remote_addr":"[fd00:1122:3344:101::b]:37412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":812,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:40.99213888Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ef6039f5-ac7f-4ecd-a0c0-ddd93b64e6e3","remote_addr":"[fd00:1122:3344:101::b]:37412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.035540603Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53269"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.035730118Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"6d9e9e60-83a3-44e8-936b-46ec4e47104d","remote_addr":"[fd00:1122:3344:101::b]:53269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.069056936Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46792"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.069792033Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"5c2c2539-679e-40b1-8498-2adb54c54ff7","remote_addr":"[fd00:1122:3344:101::c]:46792","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":702,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.090013705Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f93144d1-55ba-4172-bc83-f0afc56c2bd2","remote_addr":"[fd00:1122:3344:101::c]:61861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.286666046Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:65403"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.286852591Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.313015082Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.335709792Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"58656701-abe5-44d3-93fd-758d2a0dc04c","remote_addr":"[fd00:1122:3344:101::b]:65403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49003,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.335991128Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"172a9490-e5b3-479f-bb94-e626d3f4b7bc","remote_addr":"[fd00:1122:3344:101::b]:65403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.399627614Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42577"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.399761978Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.421613268Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.441805609Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a232fb9d-b5ab-44d0-992e-e654049022cd","remote_addr":"[fd00:1122:3344:101::c]:42577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42099,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.442119157Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"28c176eb-1773-4259-bbbc-5a0ec4836efb","remote_addr":"[fd00:1122:3344:101::c]:42577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.995790256Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:41.995916029Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2c3e47c4-13e6-4ee4-8068-5ed8c8b198de","remote_addr":"[fd00:1122:3344:101::a]:62389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":78,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:42.004495329Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4ace6092-dd10-44b0-bed8-0efb3df3d534","remote_addr":"[fd00:1122:3344:101::a]:62389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1137,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:42.197518547Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54734"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:42.197720941Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"dd8b5765-c98d-43de-b7d2-22a69bce9afc","remote_addr":"[fd00:1122:3344:101::a]:54734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":172,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:42.421564019Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49160"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:42.421708862Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:42.447397532Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:42.474025734Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1cb0d33a-3384-42af-8422-1d28190c6029","remote_addr":"[fd00:1122:3344:101::a]:49160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":52400,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:42.474340771Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9ec93343-2b7a-4909-a07a-452d12a0a6a8","remote_addr":"[fd00:1122:3344:101::a]:49160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:46.565964336Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46047"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:39:46.566051298Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f779f680-8387-4213-99e6-737225c7241b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46047","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:10.871513441Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:10.871643744Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"302f36f1-099e-41f2-8157-20b7fc1131f1","remote_addr":"[fd00:1122:3344:101::c]:54801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:10.879611871Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5e6a7cd4-1a2d-4773-8dc8-94840319c8c5","remote_addr":"[fd00:1122:3344:101::c]:54801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":130,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:10.882730473Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:10.882820215Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0818e466-c3f1-4cd3-b205-55799a856d90","remote_addr":"[fd00:1122:3344:101::b]:48181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":26,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:10.891488078Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d833306b-e85b-4c98-8d5b-29a312381a72","remote_addr":"[fd00:1122:3344:101::b]:48181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":810,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:10.937681667Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38244"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:10.937809099Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d299bf68-c77f-411e-888a-5e9f76b1a822","remote_addr":"[fd00:1122:3344:101::b]:38244","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.039613147Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.03976941Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2cef27c0-a4c4-4086-972a-9844cb3b22ae","remote_addr":"[fd00:1122:3344:101::c]:58483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.089802749Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63857"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.08987371Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.116145113Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.138926135Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"da5699cc-8fc1-4828-b9c5-3ee4fee29c34","remote_addr":"[fd00:1122:3344:101::b]:63857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49085,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.139053158Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45610"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.139231812Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.142437207Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"df612bd1-191d-4587-a2ed-a2bfddd7a223","remote_addr":"[fd00:1122:3344:101::b]:63857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3175,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.161358989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.181363716Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"670a61a9-413d-4f1b-98b6-88132edb0ec3","remote_addr":"[fd00:1122:3344:101::c]:45610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42218,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.181761865Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7e971345-1807-476d-90a1-f0a2a9819c88","remote_addr":"[fd00:1122:3344:101::c]:45610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.880232486Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53990"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.881161338Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"8f2fc300-f3ab-47b9-ba24-3f4f2467ad65","remote_addr":"[fd00:1122:3344:101::a]:53990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":904,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.916466692Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43610"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.916577755Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"32582941-70ce-4a23-a550-6f670109045f","remote_addr":"[fd00:1122:3344:101::a]:43610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":72,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:11.924302575Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f0bd3539-5649-400f-a049-d67bfcd9a71c","remote_addr":"[fd00:1122:3344:101::a]:43610","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:12.06668416Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44162"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:12.066782522Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:12.088907659Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:12.111842524Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0b679cc1-ef03-4ce3-838b-8b92376d99ee","remote_addr":"[fd00:1122:3344:101::a]:44162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:12.11210933Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"81b50823-3407-49e9-80d9-a6d09ff3127b","remote_addr":"[fd00:1122:3344:101::a]:44162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:16.599788293Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55383"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:16.599907436Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f3e9290c-c92c-406c-83fd-0ab0b5809b47","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55383","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":69,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:25.066830498Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:44106"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:25.079912614Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"6279636e-e282-4da3-a9af-0bfbdfafb846","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":12095,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:40.974586994Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:40.974688267Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"916f0460-03a9-4168-b26e-775e9e85bb86","remote_addr":"[fd00:1122:3344:101::c]:46049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:40.979586561Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:40.979643643Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"096b758c-530e-47df-8c65-c6fa60c108fd","remote_addr":"[fd00:1122:3344:101::b]:43310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":18,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:40.982075139Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"deec261d-436a-4cbb-abdc-dfb31276df94","remote_addr":"[fd00:1122:3344:101::c]:46049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:40.987026575Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f31f397b-091f-413b-b0dc-d7570dc5b723","remote_addr":"[fd00:1122:3344:101::b]:43310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":41,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.130264089Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49273"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.130418472Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d4d683a1-c9dd-4cba-8771-f0febe592b61","remote_addr":"[fd00:1122:3344:101::c]:49273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.231554894Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44485"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.231670467Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"29269c25-77e3-4d7c-acfa-df8b870a8b17","remote_addr":"[fd00:1122:3344:101::b]:44485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.316727323Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54851"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.317367528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.344921772Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.368727757Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d2d16d2e-1aad-4b10-bd19-cd904c95a655","remote_addr":"[fd00:1122:3344:101::c]:54851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":51960,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.36883854Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50181"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.368947903Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.392173385Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.414715042Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"488dc697-610f-4689-b873-85178bf24005","remote_addr":"[fd00:1122:3344:101::b]:50181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45818,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.414919576Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"dd403d86-7fa4-4d89-a0d9-16714a9f4d42","remote_addr":"[fd00:1122:3344:101::c]:54851","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.415878659Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"74bdc8fe-8001-405d-b3d7-53119264b84f","remote_addr":"[fd00:1122:3344:101::b]:50181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":843,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.8749908Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65171"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.875079342Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ecf13e89-f4df-4e49-8a71-98771a3c699b","remote_addr":"[fd00:1122:3344:101::a]:65171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.882508915Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"384877ec-0091-416c-abaa-793f58314c38","remote_addr":"[fd00:1122:3344:101::a]:65171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":127,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.908774379Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55102"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:41.908907252Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"7c37fbd2-0ab1-44dd-85f7-d3b29458c657","remote_addr":"[fd00:1122:3344:101::a]:55102","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:42.351313322Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35063"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:42.351444405Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:42.37605393Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:42.40130974Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"43de9385-d0ca-45a3-ba28-534e9a2b9801","remote_addr":"[fd00:1122:3344:101::a]:35063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49927,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:42.401591556Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b4ee0ce6-6d30-43d9-841d-ddb8422a674b","remote_addr":"[fd00:1122:3344:101::a]:35063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:46.633525257Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:46.633629149Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"1b416c65-b104-4ff8-aed0-45a0c8c6a0d3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52915","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":78,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:40:55.066600778Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"dd2c762e-d39f-48d9-98ba-41e2e6fecdc4","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":597,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.893470693Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.893591796Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"23934f21-7583-461e-b65c-255e162016cf","remote_addr":"[fd00:1122:3344:101::b]:50758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.90061076Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"fddd23a5-ac02-4554-b431-4d13ecec363d","remote_addr":"[fd00:1122:3344:101::b]:50758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.913506241Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.913667085Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"9744a675-412e-4e1f-8675-9e9569a3c3c7","remote_addr":"[fd00:1122:3344:101::b]:55264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.934565343Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57929"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.935509545Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"6c19d354-7c31-4f3d-97ff-a1d7b11973ab","remote_addr":"[fd00:1122:3344:101::c]:57929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":908,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.976474672Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.976616585Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b57b6ba8-42fa-4ea0-8743-17580a4f70c9","remote_addr":"[fd00:1122:3344:101::c]:49217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":80,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:10.984677533Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4c2d101d-c7f0-4cf1-b8ba-8a0d544b326f","remote_addr":"[fd00:1122:3344:101::c]:49217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.045250798Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56173"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.0453658Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.071190514Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.093974026Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e699c91f-af57-41e5-ad79-dbf172645162","remote_addr":"[fd00:1122:3344:101::b]:56173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48662,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.094080668Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44544"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.094714123Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.119211004Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.143279546Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"441c3881-f54b-449c-a9c9-dfdb83aa51bf","remote_addr":"[fd00:1122:3344:101::c]:44544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48600,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.143572773Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f091545e-480e-47a6-bf14-86c9f262cc9e","remote_addr":"[fd00:1122:3344:101::b]:56173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.144342871Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e807d624-4903-48c5-8f57-db7cea2bf48f","remote_addr":"[fd00:1122:3344:101::c]:44544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":667,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.848222378Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.848417332Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"dde749c4-2ead-46f8-a2af-0a193bba0947","remote_addr":"[fd00:1122:3344:101::a]:34281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.901260746Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62101"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.901356279Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"bc4e8211-fa0d-487d-a527-bfc1278d8f78","remote_addr":"[fd00:1122:3344:101::a]:62101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:11.909698834Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f9d5cd20-4cb4-44d2-80f1-80f878ae170d","remote_addr":"[fd00:1122:3344:101::a]:62101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:12.06242438Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37459"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:12.062496322Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:12.085778265Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:12.109287214Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"aefcdfa0-c271-4e30-bb1b-87f928a94e02","remote_addr":"[fd00:1122:3344:101::a]:37459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46825,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:12.109594991Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4c15f30f-ce38-46f9-9a32-a2831a35e5a9","remote_addr":"[fd00:1122:3344:101::a]:37459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:16.667514714Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45078"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:16.667602706Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"29a9bad0-0088-4322-8a3e-89887c21fdb8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45078","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":76,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:25.065946398Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"6448806a-0a2f-4d62-826d-7642eb73bdeb","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":528,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:40.961942639Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64212"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:40.962924002Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"96f74ebb-0824-46dc-af18-9f071fcfe82d","remote_addr":"[fd00:1122:3344:101::c]:64212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":958,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:40.97010111Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"1d1692c9-55e1-434a-be4a-6d5d03098ee1","remote_addr":"[fd00:1122:3344:101::c]:64212","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.109530186Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.109674649Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"306c0500-49b8-46b7-bdd2-cc87518ecf9e","remote_addr":"[fd00:1122:3344:101::c]:48324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.127342191Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52202"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.127494815Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"daa96d31-9ea0-430e-8940-615e97d8ac44","remote_addr":"[fd00:1122:3344:101::b]:52202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.143069239Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61367"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.14312755Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2c89e017-8e89-4f0c-b014-ae9e55d1ffb7","remote_addr":"[fd00:1122:3344:101::b]:61367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":25,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.151025695Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"7ae047d7-83dd-4d22-8ff6-d84ba71e271f","remote_addr":"[fd00:1122:3344:101::b]:61367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":79,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.294365232Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60129"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.294439193Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.316427297Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.339129637Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1d7437fa-3eab-4800-8e55-168f54a2ba23","remote_addr":"[fd00:1122:3344:101::c]:60129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":44727,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.33926451Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37847"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.339497706Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.342741712Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5225e2d5-c2a9-466f-ac9a-d6009fcd810b","remote_addr":"[fd00:1122:3344:101::c]:60129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3234,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.364351326Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.386225527Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a345ae03-4f4d-4d53-a6f5-0968d93ba831","remote_addr":"[fd00:1122:3344:101::b]:37847","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46830,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.387161359Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"95a53a6e-cf25-4b0f-b790-ebbaa5ee9f57","remote_addr":"[fd00:1122:3344:101::b]:37847","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":736,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.857298068Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58260"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.85742434Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"419f7801-f579-43ec-b2a4-3d2c588b0d4c","remote_addr":"[fd00:1122:3344:101::a]:58260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.865471458Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"47ddf004-a3a3-49d7-ba87-05e4d440206e","remote_addr":"[fd00:1122:3344:101::a]:58260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":142,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.978064568Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:41.978917488Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ee05eddb-e362-4f8e-a37c-648ba69262f1","remote_addr":"[fd00:1122:3344:101::a]:53158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":825,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:42.142608379Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60069"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:42.143260014Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:42.165030283Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:42.185574962Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"32fe988e-8d93-4ec5-8768-8f65822f74f4","remote_addr":"[fd00:1122:3344:101::a]:60069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42930,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:42.185845729Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0c77ab8c-4327-4d3a-8413-401b0dfd1752","remote_addr":"[fd00:1122:3344:101::a]:60069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:46.701133637Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:46.701241709Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"90ec957d-96bd-4e71-b583-46f1db03dc47","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46018","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:41:55.065863513Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"3a95b168-a2f0-45be-9558-4087885ab5c4","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":617,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:10.913925985Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43399"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:10.914046538Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"76e9e2ed-56e8-4b59-b3b8-2cc66f91c9ff","remote_addr":"[fd00:1122:3344:101::c]:43399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":85,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:10.921495682Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"135aa72d-157f-4248-92a1-176d82c09ecd","remote_addr":"[fd00:1122:3344:101::c]:43399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:10.923469348Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50633"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:10.923594801Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f6896eb9-5039-49fe-a39f-9642d0776c14","remote_addr":"[fd00:1122:3344:101::c]:50633","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:10.931971697Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:10.932078799Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ec59ef0e-42e2-43a0-9541-152053d95489","remote_addr":"[fd00:1122:3344:101::b]:64173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:10.939467852Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"2dbc4125-84b7-4a25-97e2-12539e4be2c7","remote_addr":"[fd00:1122:3344:101::b]:64173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":64,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.018532538Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.018660101Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2727f9f5-2b53-4686-8e98-ab7a836aad41","remote_addr":"[fd00:1122:3344:101::b]:33476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.102754444Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39085"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.103513702Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.125931485Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.144065109Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"795408f8-4318-4744-92d9-07023b7f2d3f","remote_addr":"[fd00:1122:3344:101::c]:39085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":41257,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.144342995Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"42a1e23d-6a08-482e-b3ef-5ee71603d5e1","remote_addr":"[fd00:1122:3344:101::c]:39085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.219776697Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53988"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.219966371Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.244496244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.265511814Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a9ba5d2d-f481-4b06-b826-0f34da73316a","remote_addr":"[fd00:1122:3344:101::b]:53988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45628,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.26575785Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"57ce4c1a-1354-414d-9d3a-39d1cd29bc89","remote_addr":"[fd00:1122:3344:101::b]:53988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.765314676Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.765458619Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"3b6ad828-d0d1-40ac-bcd4-7810b1e90b82","remote_addr":"[fd00:1122:3344:101::a]:58722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.773370554Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"03c82061-d404-483e-bc5e-cb6988337054","remote_addr":"[fd00:1122:3344:101::a]:58722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.79118853Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54815"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.791345324Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"7dca0f7e-0177-4347-a6f2-cd3cea63038d","remote_addr":"[fd00:1122:3344:101::a]:54815","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.895911295Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50487"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.896009428Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.920630333Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.941407298Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"98a2776e-7f20-4fcd-a819-2362a114196d","remote_addr":"[fd00:1122:3344:101::a]:50487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45444,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:11.941641523Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b1d2a232-0975-4db6-8b02-dd65b18cbf03","remote_addr":"[fd00:1122:3344:101::a]:50487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:16.746101654Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:16.746828641Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"469f7fc5-4280-42a2-9a81-c59cea3e65b4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42620","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:25.066083236Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"030f0bf6-fda3-46ad-8d6c-2273e51cfa40","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":565,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:40.981429408Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:40.98150823Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a79f423e-a9fb-45a6-83a4-385717970c27","remote_addr":"[fd00:1122:3344:101::b]:48421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":54,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:40.988141695Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"657dc0a8-9677-41a3-a37b-ffc152a13484","remote_addr":"[fd00:1122:3344:101::b]:48421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.128017301Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62987"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.129402583Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"6479262e-ad14-408c-bf45-456134022c8e","remote_addr":"[fd00:1122:3344:101::b]:62987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1362,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.142665493Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40050"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.142725084Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8d525d2e-e0b4-47df-8276-6bfe8acae3e1","remote_addr":"[fd00:1122:3344:101::c]:40050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":28,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.150403643Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"8da3ae22-d783-4ddf-957f-05d258c097b3","remote_addr":"[fd00:1122:3344:101::c]:40050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.193972341Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47977"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.194106294Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d76851cf-53b6-44c7-a8f7-adfd1e474afb","remote_addr":"[fd00:1122:3344:101::c]:47977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.296868604Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47121"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.296981386Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.321640882Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.344532427Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8f2734d4-c93d-4a6a-90a0-89b95f6cf772","remote_addr":"[fd00:1122:3344:101::b]:47121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47598,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.345447678Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a6c26da6-e2d5-48f7-8d9e-9ebe15aad411","remote_addr":"[fd00:1122:3344:101::b]:47121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":739,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.382992115Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50434"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.383151669Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.405806327Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.42688395Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"93348065-bc49-4d9b-9c75-bd210a9aa67b","remote_addr":"[fd00:1122:3344:101::c]:50434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43795,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.427132916Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d974a21d-84f8-4ce4-a8a1-ca869094a297","remote_addr":"[fd00:1122:3344:101::c]:50434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.873874778Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37520"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.874002161Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"5007455c-4c78-4ca7-8656-970abcfbbe65","remote_addr":"[fd00:1122:3344:101::a]:37520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.880797369Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ea08c726-0830-48ba-93ed-9c53a3a7f126","remote_addr":"[fd00:1122:3344:101::a]:37520","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":137,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.921231374Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62579"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:41.921388877Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0621b4a7-98a0-47d6-a199-e7cc4202b5f1","remote_addr":"[fd00:1122:3344:101::a]:62579","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:42.203650918Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48642"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:42.203758Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:42.229481741Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:42.253165834Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b263be18-954c-4fd6-aede-d8952e56d22c","remote_addr":"[fd00:1122:3344:101::a]:48642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49461,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:42.25343759Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"779f0f7b-1953-4dc0-af01-707b574edf1e","remote_addr":"[fd00:1122:3344:101::a]:48642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:46.781018314Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46229"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:46.781585577Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"1fde6376-ece6-4702-8801-bfe788631212","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46229","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":550,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:42:55.121907973Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"03541c1d-7f11-46f7-84e4-b9132d363aee","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":593,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:10.913965337Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65025"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:10.914218113Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f59ccf9e-b9e5-45c6-9f93-583e5cd5d516","remote_addr":"[fd00:1122:3344:101::c]:65025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":213,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:10.921286608Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47355"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:10.921406881Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"98ddd58d-46a5-40e1-91c3-87b4575e7a43","remote_addr":"[fd00:1122:3344:101::b]:47355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:10.928144158Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"35c79077-9874-4489-95ce-52500ed88a49","remote_addr":"[fd00:1122:3344:101::b]:47355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":137,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:10.95091849Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:49072"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:10.951056333Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"c905438a-7acd-4c24-9c23-d8dddb3b22d3","remote_addr":"[fd00:1122:3344:101::b]:49072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.008893664Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47770"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.008957265Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cb3d502c-10b7-4f38-bf0e-eab87e04da56","remote_addr":"[fd00:1122:3344:101::c]:47770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.017531355Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"70932880-d38b-4f1a-8bcd-aba3ce97e1d0","remote_addr":"[fd00:1122:3344:101::c]:47770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.119188828Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39650"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.11925076Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.143094457Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.166737719Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fedd0c2c-0293-495d-aadc-9da869b57016","remote_addr":"[fd00:1122:3344:101::c]:39650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47509,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.166875202Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43407"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.166990485Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.189885479Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.211989046Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"13674255-eed5-4146-b40c-364346732ef6","remote_addr":"[fd00:1122:3344:101::b]:43407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45048,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.212250482Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8d7b9b4a-d276-4f2c-bfaa-7e81e9bafa3e","remote_addr":"[fd00:1122:3344:101::c]:39650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.212474777Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"82c07c30-c5d8-46e7-b578-707fb11f7018","remote_addr":"[fd00:1122:3344:101::b]:43407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.793114836Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39477"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.793252539Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"dc572608-d2d7-4a3b-993f-ab8c750e13ca","remote_addr":"[fd00:1122:3344:101::a]:39477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.800049038Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"608b930b-508f-48bb-80e5-183edda7ff1c","remote_addr":"[fd00:1122:3344:101::a]:39477","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.837803309Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57976"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:11.837967743Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"16482ab9-a719-42dc-9ae5-cfbac45c5ceb","remote_addr":"[fd00:1122:3344:101::a]:57976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:12.002616258Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49998"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:12.003300474Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:12.030469139Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:12.053037206Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"db30d9c9-55ea-4121-b33f-a3880caf20ab","remote_addr":"[fd00:1122:3344:101::a]:49998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50368,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:12.053298752Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ad2dea24-74ec-4243-a737-62ba7f128c0a","remote_addr":"[fd00:1122:3344:101::a]:49998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:16.813892637Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:16.814001699Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"59574a84-b367-4ce1-935f-9161dab1d56f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54154","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:25.124714612Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"96b1bded-fad6-49a4-bcc7-6541d11d3d22","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:40.968878755Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48687"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:40.969026299Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cdc801a6-e0b7-4eb9-b3ad-a98507889993","remote_addr":"[fd00:1122:3344:101::c]:48687","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:40.969307945Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41189"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:40.970584435Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"1dc21a5a-513d-44f3-82b0-50b2da821963","remote_addr":"[fd00:1122:3344:101::c]:41189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1226,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:40.976443062Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4bc4c800-77b9-44f3-a4f8-f55df839f6a1","remote_addr":"[fd00:1122:3344:101::c]:48687","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":127,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.12240751Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40715"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.122577794Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2f46bf86-4fc6-46a8-b573-bf2fb626f25d, kind: Service { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425 }, name: Name(\"external-dns-cf7587f8-33d7-4e89-bad7-ae8cdd556425\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 111])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 567092fa-fb08-4940-8635-631e5c908557, kind: Service { id: 53979846-b429-4bae-9968-d96ff860d07e }, name: Name(\"external-dns-53979846-b429-4bae-9968-d96ff860d07e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 230])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 824ae30e-19ba-423f-923a-3afa2a59b03f, kind: Service { id: 8b4031b8-df71-45f8-b977-b6aa00214087 }, name: Name(\"nexus-8b4031b8-df71-45f8-b977-b6aa00214087\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 145])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95c06611-4118-4e93-84ac-1b68f8ff2a6d, kind: Service { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087 }, name: Name(\"nexus-9d80bc64-2ec6-4d4e-96d7-5c012d075087\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e94e36b7-3a4e-4066-8a04-518f25a7ffdb, kind: Service { id: 56f1d4b9-214d-4104-bc80-a770b9be089e }, name: Name(\"nexus-56f1d4b9-214d-4104-bc80-a770b9be089e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 23])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.122608344Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.122646065Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.122664125Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.122700046Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.122710527Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.122754018Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.122768608Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"069f380f-c8bc-4ea3-a02e-2f40735f774f","remote_addr":"[fd00:1122:3344:101::c]:40715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":331,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.148647202Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40220"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.148745525Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"668cf6e1-63db-4468-8ef4-196e423cf474","remote_addr":"[fd00:1122:3344:101::b]:40220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":29,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.155786299Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ccd4db0a-c9e4-49e8-afc1-c23a546e95a6","remote_addr":"[fd00:1122:3344:101::b]:40220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":64,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.186942857Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41762"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.18706474Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"3c7c3272-dec3-4327-97af-4bf8aaf7229b","remote_addr":"[fd00:1122:3344:101::b]:41762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.311997087Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33544"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.312095709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.336349095Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.359973047Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"dc7e15cf-c1b6-49a1-872b-31c27263939b","remote_addr":"[fd00:1122:3344:101::c]:33544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47906,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.3601015Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59194"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.360462679Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2f46bf86-4fc6-46a8-b573-bf2fb626f25d, kind: Service { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425 }, name: Name(\"external-dns-cf7587f8-33d7-4e89-bad7-ae8cdd556425\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 111])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 567092fa-fb08-4940-8635-631e5c908557, kind: Service { id: 53979846-b429-4bae-9968-d96ff860d07e }, name: Name(\"external-dns-53979846-b429-4bae-9968-d96ff860d07e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 230])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 824ae30e-19ba-423f-923a-3afa2a59b03f, kind: Service { id: 8b4031b8-df71-45f8-b977-b6aa00214087 }, name: Name(\"nexus-8b4031b8-df71-45f8-b977-b6aa00214087\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 145])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95c06611-4118-4e93-84ac-1b68f8ff2a6d, kind: Service { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087 }, name: Name(\"nexus-9d80bc64-2ec6-4d4e-96d7-5c012d075087\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e94e36b7-3a4e-4066-8a04-518f25a7ffdb, kind: Service { id: 56f1d4b9-214d-4104-bc80-a770b9be089e }, name: Name(\"nexus-56f1d4b9-214d-4104-bc80-a770b9be089e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 23])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.36052402Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.36053538Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ca5ef92a-b61b-4f54-8577-33d71feabf8b","remote_addr":"[fd00:1122:3344:101::c]:33544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":265,"response_code":"200"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.360579321Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.360604152Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.360624842Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.360638263Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.360657213Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.360672254Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"32fc2e99-c69f-4f3e-a8a7-38e3626a47bc","remote_addr":"[fd00:1122:3344:101::b]:59194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":492,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.436977895Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43406"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.437644971Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.460915014Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.48212428Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5436d855-944d-4274-b0c4-94bfe01f3676","remote_addr":"[fd00:1122:3344:101::b]:43406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45073,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.482382826Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8a498377-a1bc-49d7-a99b-642136f15312","remote_addr":"[fd00:1122:3344:101::b]:43406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.850328308Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37153"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.850475091Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d6f5a590-70ea-4cbe-b7a0-69aac3e66eb8","remote_addr":"[fd00:1122:3344:101::a]:37153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.857208269Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d06cc4f3-3ee8-4892-9347-2776f3e0539b","remote_addr":"[fd00:1122:3344:101::a]:37153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":130,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.908633299Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45439"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:41.908878055Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"3be8c615-5ac0-4ff2-b5ff-2dcae7a5624d","remote_addr":"[fd00:1122:3344:101::a]:45439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":194,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007378615Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36479"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007617301Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2f46bf86-4fc6-46a8-b573-bf2fb626f25d, kind: Service { id: cf7587f8-33d7-4e89-bad7-ae8cdd556425 }, name: Name(\"external-dns-cf7587f8-33d7-4e89-bad7-ae8cdd556425\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 168, 111])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 567092fa-fb08-4940-8635-631e5c908557, kind: Service { id: 53979846-b429-4bae-9968-d96ff860d07e }, name: Name(\"external-dns-53979846-b429-4bae-9968-d96ff860d07e\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 230])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 824ae30e-19ba-423f-923a-3afa2a59b03f, kind: Service { id: 8b4031b8-df71-45f8-b977-b6aa00214087 }, name: Name(\"nexus-8b4031b8-df71-45f8-b977-b6aa00214087\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 161, 145])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95c06611-4118-4e93-84ac-1b68f8ff2a6d, kind: Service { id: 9d80bc64-2ec6-4d4e-96d7-5c012d075087 }, name: Name(\"nexus-9d80bc64-2ec6-4d4e-96d7-5c012d075087\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 248, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e94e36b7-3a4e-4066-8a04-518f25a7ffdb, kind: Service { id: 56f1d4b9-214d-4104-bc80-a770b9be089e }, name: Name(\"nexus-56f1d4b9-214d-4104-bc80-a770b9be089e\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 180, 23])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007672072Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007709883Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007723493Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007740934Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007748134Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007766265Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.007798065Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7a6d753f-b670-4d67-9f59-311297329cdb","remote_addr":"[fd00:1122:3344:101::a]:36479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":339,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.362788754Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35112"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.366546782Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.388832012Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.411895181Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3191f10c-29ca-40d2-8e9f-66be6e1e9f2c","remote_addr":"[fd00:1122:3344:101::a]:35112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49051,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:42.412928805Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0628f76c-a6ce-4c22-81b7-a776ce8c9c08","remote_addr":"[fd00:1122:3344:101::a]:35112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":884,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:46.846504434Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54286"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:46.846593286Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"d3c05be1-f207-4047-9c28-9d8f0c19b2b3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54286","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:55.12340775Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"a7c4d2bf-8800-49b7-a19f-42a4fffcd23b","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2352,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:55.850621901Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:303"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:55.851707547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:43:55.890375Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.9999979s","next_cleanup":"Instant { tv_sec: 13394, tv_nsec: 853998981 }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:09.765073471Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:10.936179538Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:10.936391243Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"aafa3518-f113-4161-b763-d52a6f2100fb","remote_addr":"[fd00:1122:3344:101::b]:55493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:10.995623726Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53708"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:10.99578194Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"5f75a1e1-1b95-4075-bdd9-b1c6a1b1b7b5","remote_addr":"[fd00:1122:3344:101::c]:53708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.01333844Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42559"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.013423362Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e57330f0-531b-4a49-b975-aabaf44b5b59","remote_addr":"[fd00:1122:3344:101::c]:42559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.016234677Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.016285209Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"3c1264e3-bfe2-4038-b876-aad2d27e0515","remote_addr":"[fd00:1122:3344:101::b]:55310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":22,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.022174676Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"864f24a2-d560-44a9-9bf5-931f1ee7e969","remote_addr":"[fd00:1122:3344:101::c]:42559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.023737092Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"1019cace-57ed-4f19-a1a5-6474949d5b3b","remote_addr":"[fd00:1122:3344:101::b]:55310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.103330701Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41440"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.103940255Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.129790889Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.152849098Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0aeb01ff-49bc-48bd-b408-539c4bf310a9","remote_addr":"[fd00:1122:3344:101::b]:41440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49473,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.153099823Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49093"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.153415541Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"39eac491-0c08-46cf-b988-9f64c645aabe","remote_addr":"[fd00:1122:3344:101::b]:41440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":278,"response_code":"200"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.154172958Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.179447328Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.201171066Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c50724a0-1ccf-471f-b72b-84ee2b71e3bf","remote_addr":"[fd00:1122:3344:101::c]:49093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47914,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.201575245Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"be9daae2-842b-4870-b433-226c954a7d05","remote_addr":"[fd00:1122:3344:101::c]:49093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.922299846Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48232"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.922841908Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"27c4448f-9331-4584-8633-81202f1d5d80","remote_addr":"[fd00:1122:3344:101::a]:48232","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":511,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.92764477Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.927879196Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ef8f7b49-a672-4fdc-b45f-1a9391a3e169","remote_addr":"[fd00:1122:3344:101::a]:55034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":171,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:11.933276202Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f6bdbf7d-efc9-4e31-9f77-ca02e4a516e9","remote_addr":"[fd00:1122:3344:101::a]:48232","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":128,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:12.003136203Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60523"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:12.003230396Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:12.029493349Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:12.053010768Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a0142b38-ebfc-4e8d-a2fc-93a05b8824bc","remote_addr":"[fd00:1122:3344:101::a]:60523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49833,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:12.053334506Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5db2350c-becc-44cf-99b7-2279e8d7ecb2","remote_addr":"[fd00:1122:3344:101::a]:60523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:16.88027641Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42778"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:16.880400083Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6d04a316-9e41-4dfa-8b78-d1d58c164dc6","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42778","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:25.122820403Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"e329470f-192c-45e9-a375-4f4165eb7fcb","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1557,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:40.945546464Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48521"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:40.945661336Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0e825475-3bd8-426b-bfb4-57192ba8cd24","remote_addr":"[fd00:1122:3344:101::c]:48521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:40.95311166Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"17671cd8-066f-4b36-b034-477d4f3e98f0","remote_addr":"[fd00:1122:3344:101::c]:48521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":137,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:40.97321698Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:40.973294362Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a39c587f-d771-4d9d-9611-2f35bc2daca9","remote_addr":"[fd00:1122:3344:101::b]:36675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:40.980474379Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"afcbdc43-28ea-4d98-ab6c-1764b7c5e19e","remote_addr":"[fd00:1122:3344:101::b]:36675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:40.982536298Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40986"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:40.982682171Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"bce3d7db-a7a5-47ef-a118-d4fa0d4f3676","remote_addr":"[fd00:1122:3344:101::b]:40986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.100133893Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64355"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.100368549Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"a9960d91-1f75-4abe-8366-f61a22892038","remote_addr":"[fd00:1122:3344:101::c]:64355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.259186577Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65149"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.259967976Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.28455191Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.309270297Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5cebf478-a3b8-464a-8036-4155d8a494d7","remote_addr":"[fd00:1122:3344:101::c]:65149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50016,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.309447411Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34393"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.309625895Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.313317341Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"72b98242-d659-42c5-8322-c755a43cb898","remote_addr":"[fd00:1122:3344:101::c]:65149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3719,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.335804706Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.358891466Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a15d2f91-d419-4a70-b265-29f50a99eec1","remote_addr":"[fd00:1122:3344:101::b]:34393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49362,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:41.359318165Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4023b80a-1299-4ad8-bf57-a9d4c10de6b1","remote_addr":"[fd00:1122:3344:101::b]:34393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.009819716Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61359"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.010022181Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"a6c36d50-1ae0-43b8-a322-59bd9bcdec79","remote_addr":"[fd00:1122:3344:101::a]:61359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.130165505Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55780"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.130311919Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e3ca3040-5779-4320-b447-4f5888d2f17a","remote_addr":"[fd00:1122:3344:101::a]:55780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.137877276Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"aaaca2fa-447b-43bb-aaaf-03ed686a9883","remote_addr":"[fd00:1122:3344:101::a]:55780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.253132617Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38643"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.253204049Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.278988341Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.301762883Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"bed611fd-093e-4fc1-9746-bfb4cedcaf50","remote_addr":"[fd00:1122:3344:101::a]:38643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48591,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:42.301993468Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ab26bf28-32bd-45ac-91c4-eddd4fdaa225","remote_addr":"[fd00:1122:3344:101::a]:38643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:46.914090738Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58033"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:46.914252981Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c906c170-3204-4979-8779-0f34c1be29df","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58033","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:44:55.121743107Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"e68c9a8e-1ede-4024-8cb8-3de6fb494da9","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":598,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:10.908259562Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46706"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:10.908400866Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cd884401-43cd-461e-a6ad-61dd764219f2","remote_addr":"[fd00:1122:3344:101::c]:46706","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:10.910340671Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47147"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:10.910501315Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"b5f8f034-5850-446c-bc44-5af28638fb54","remote_addr":"[fd00:1122:3344:101::b]:47147","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:10.913977076Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50056"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:10.914103229Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"46a3b4af-f623-47cc-9b80-1b7581a14ccf","remote_addr":"[fd00:1122:3344:101::c]:50056","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":89,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:10.91589587Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"43c75a84-197f-43dd-9590-8dced295be95","remote_addr":"[fd00:1122:3344:101::c]:46706","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":107,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.005692587Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33567"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.006365793Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"5ead8e98-ae1f-41d1-9fb8-ba09f7680b99","remote_addr":"[fd00:1122:3344:101::b]:33567","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":655,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.013115071Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"8126ab43-297f-4d20-8ebe-6f36a62d7dae","remote_addr":"[fd00:1122:3344:101::b]:33567","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":83,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.118105472Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41468"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.118175793Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.146729Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.167593407Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0644b5e5-ce87-4576-9757-e14afaa78e86","remote_addr":"[fd00:1122:3344:101::c]:41468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49452,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.168345865Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"59dde202-1215-4270-9969-eea04e87e7e7","remote_addr":"[fd00:1122:3344:101::c]:41468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":567,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.212040725Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36680"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.212207999Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.235934683Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.255415288Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"39749632-01de-4216-8fd4-4ce72159e3e9","remote_addr":"[fd00:1122:3344:101::b]:36680","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43276,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.256294859Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"dc19a9ac-5f55-4bc2-bf59-573e566a42cf","remote_addr":"[fd00:1122:3344:101::b]:36680","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":700,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.813674905Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41305"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.813782977Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"fededbeb-067d-4429-92bc-3338941798e9","remote_addr":"[fd00:1122:3344:101::a]:41305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.819316336Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43964"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.81946779Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0f865705-303e-41eb-ad4b-fecaa939c743","remote_addr":"[fd00:1122:3344:101::a]:43964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":132,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:11.820902674Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"1ed3ccdc-613b-49b4-99cb-e2e3400aff1f","remote_addr":"[fd00:1122:3344:101::a]:41305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":103,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:12.026561246Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59197"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:12.026655648Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:12.051485868Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:12.074800852Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1d5c09aa-1c88-4109-8309-d0b32b1736dc","remote_addr":"[fd00:1122:3344:101::a]:59197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48196,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:12.075066609Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"faa03a72-c1e8-451e-a211-18a9abec5fd0","remote_addr":"[fd00:1122:3344:101::a]:59197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.083341124Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53590"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.084118422Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"a0edf1d0-b1b2-4dac-9756-90d9981682db","remote_addr":"[fd00:1122:3344:101::a]:53590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":754,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.084912581Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9883005) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.084946262Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"bf2ddbbe-625b-48b2-85b8-f1b7e723e92d","remote_addr":"[fd00:1122:3344:101::a]:53590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":612,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.264968686Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48728"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.311121623Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:577","metadata":"InstanceMetadata { silo_id: 2218f0bf-bef6-4785-b5b0-f8053a24df0b, project_id: c67ecc4c-8545-4ca9-94c2-44f13e076acb }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-26T02:45:13.695276Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-26T02:45:13.770859467Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 1c18881b-2d0d-40fc-b0ae-bff2888c2686, kind: Instance { id: 4339d003-d6c5-45d3-997b-9e16d27048e0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9883005), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c18881b-2d0d-40fc-b0ae-bff2888c2686, kind: Instance { id: 4339d003-d6c5-45d3-997b-9e16d27048e0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9883005), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c18881b-2d0d-40fc-b0ae-bff2888c2686, kind: Instance { id: 4339d003-d6c5-45d3-997b-9e16d27048e0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9883005), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(9883005))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c18881b-2d0d-40fc-b0ae-bff2888c2686, kind: Instance { id: 4339d003-d6c5-45d3-997b-9e16d27048e0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9883005), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-9a55edbf7aa\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 93202239-8905-4143-90c9-72c9c2c57851, read_only_parent: Some(Volume { block_size: 512, id: e987411a-338f-4162-9be3-96b3a0db18dc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a6916a24-7388-4413-b242-8924c309b31f, key: Some(\"R/SR4dIKzzlXrmeLKtmZK2t3475bq4aJM1yuNV7xByE=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 93202239-8905-4143-90c9-72c9c2c57851, key: Some(\"jmeSKXpRneOBscv+9emybfwWLqgljhgjMS1FaYBUuH4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAVpxX6WPpYAACnFfpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAFqcV+lj6WAAApxX6WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MzM5ZDAwMy1kNmM1LTQ1ZDMtOTk3Yi05ZTE2ZDI3MDQ4ZTAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTVRkTE8zclQ3Uy90d2oydmY1Y3hvK3NlUTM1cG1nbXowV2loeU1QajJIZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.311228526Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:611","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0 (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.311363499Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:951","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 1c18881b-2d0d-40fc-b0ae-bff2888c2686, kind: Instance { id: 4339d003-d6c5-45d3-997b-9e16d27048e0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9883005), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c18881b-2d0d-40fc-b0ae-bff2888c2686, kind: Instance { id: 4339d003-d6c5-45d3-997b-9e16d27048e0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9883005), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c18881b-2d0d-40fc-b0ae-bff2888c2686, kind: Instance { id: 4339d003-d6c5-45d3-997b-9e16d27048e0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9883005), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(9883005))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c18881b-2d0d-40fc-b0ae-bff2888c2686, kind: Instance { id: 4339d003-d6c5-45d3-997b-9e16d27048e0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9883005), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-9a55edbf7aa\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 93202239-8905-4143-90c9-72c9c2c57851, read_only_parent: Some(Volume { block_size: 512, id: e987411a-338f-4162-9be3-96b3a0db18dc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a6916a24-7388-4413-b242-8924c309b31f, key: Some(\"R/SR4dIKzzlXrmeLKtmZK2t3475bq4aJM1yuNV7xByE=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 93202239-8905-4143-90c9-72c9c2c57851, key: Some(\"jmeSKXpRneOBscv+9emybfwWLqgljhgjMS1FaYBUuH4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAVpxX6WPpYAACnFfpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAFqcV+lj6WAAApxX6WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MzM5ZDAwMy1kNmM1LTQ1ZDMtOTk3Yi05ZTE2ZDI3MDQ4ZTAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTVRkTE8zclQ3Uy90d2oydmY1Y3hvK3NlUTM1cG1nbXowV2loeU1QajJIZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-26T02:45:13.770859467Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-26T02:45:13.695276Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.312610178Z","hostname":"sock","pid":1555,"uri":"/instances/4339d003-d6c5-45d3-997b-9e16d27048e0","method":"PUT","req_id":"6fef7c6e-77ac-49f0-9cd3-f787052d03b1","remote_addr":"[fd00:1122:3344:101::a]:48728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47595,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.569931857Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.574136355Z","hostname":"sock","pid":1555,"uri":"/instances/4339d003-d6c5-45d3-997b-9e16d27048e0/state","method":"PUT","req_id":"746f27f1-2068-4c9a-98c5-53a199250169","remote_addr":"[fd00:1122:3344:101::a]:35205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":4089,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.578109478Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 254, 166, 16]), slot: 0, vni: Vni { inner: 9883005 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.578718912Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.612069321Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:14.634375332Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:16.328796188Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:16.328961262Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 38.46868ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:16.502022683Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 68.56394ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:16.549638035Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 278.034185ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:16.630690818Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 342.866818ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:16.925134564Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:16.96008068Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50127"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:16.960992941Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"40b447b4-e6ad-4d5d-9ae5-37c9be72dd8a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50127","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":823,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 683.412523ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:17.282089819Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 1.489075298s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:17.976452674Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 765.708939ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:19.475686362Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 1.441382496s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:20.25030823Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 1.098852606s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:21.702557562Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:22.829260762Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:1460"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 30.487007ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:22.837360651Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 131.49786ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:22.877201441Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 223.706481ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:23.018851579Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 338.16019ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:23.253516598Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\") failed: Property not found. retry in 1.028576976s","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:23.601952485Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.643727721Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:1469"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.68004949Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:1483"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAVpxX6WPpYAACnFfpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAFqcV+lj6WAAApxX6WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MzM5ZDAwMy1kNmM1LTQ1ZDMtOTk3Yi05ZTE2ZDI3MDQ4ZTAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTVRkTE8zclQ3Uy90d2oydmY1Y3hvK3NlUTM1cG1nbXowV2loeU1QajJIZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-9a55edbf7aa\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 93202239-8905-4143-90c9-72c9c2c57851, read_only_parent: Some(Volume { block_size: 512, id: e987411a-338f-4162-9be3-96b3a0db18dc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a6916a24-7388-4413-b242-8924c309b31f, key: Some(\"R/SR4dIKzzlXrmeLKtmZK2t3475bq4aJM1yuNV7xByE=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 93202239-8905-4143-90c9-72c9c2c57851, key: Some(\"jmeSKXpRneOBscv+9emybfwWLqgljhgjMS1FaYBUuH4=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"opte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 4339d003-d6c5-45d3-997b-9e16d27048e0, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: c67ecc4c-8545-4ca9-94c2-44f13e076acb, silo_id: 2218f0bf-bef6-4785-b5b0-f8053a24df0b }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.682036956Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:698"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.825799333Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:700"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-07-26T02:45:24.829362576Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.830162645Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.830177185Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-26T02:45:24.829362576Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-26T02:45:13.770859467Z }","propolis_id":"400431bd-e96b-4ea7-9e2f-5a4d556819e4"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.830194006Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-26T02:45:13.770859467Z }, propolis_id: 400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-26T02:45:24.829362576Z }, migration_state: None }","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-07-26T02:45:24.924643231Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.924663022Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.924683392Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-26T02:45:24.924643231Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-26T02:45:13.770859467Z }","propolis_id":"400431bd-e96b-4ea7-9e2f-5a4d556819e4"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.924710723Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-26T02:45:13.770859467Z }, propolis_id: 400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-26T02:45:24.924643231Z }, migration_state: None }","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.962566847Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42422"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:24.96268694Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ad8ab023-ec70-46eb-acd8-476bc6fe45bb","remote_addr":"[fd00:1122:3344:101::a]:42422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:25.013459075Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b9ee1c0e-c282-406d-ab30-d56de3dd36d9","remote_addr":"[fd00:1122:3344:101::a]:42422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":210,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:25.057060363Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:25.057132855Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c734e07d-515e-4567-aaa8-92858888b646","remote_addr":"[fd00:1122:3344:101::a]:43281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:25.069472423Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4d7e384a-f1af-4e95-aa44-06f7cacece05","remote_addr":"[fd00:1122:3344:101::a]:43281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:25.203928832Z","hostname":"sock","pid":1555,"uri":"/ea0e0fc4-fc93-4de6-b687-98567d824e83","method":"GET","req_id":"816ff992-8d15-4c33-ad89-b1bc5cc0cb8e","remote_addr":"[fd00:1122:3344:101::d]:44106","local_addr":"[fd00:1122:3344:101::1]:33355","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"ea0e0fc4-fc93-4de6-b687-98567d824e83","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":736,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.070225445Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54567"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.071231938Z","hostname":"sock","pid":1555,"uri":"/instances/4339d003-d6c5-45d3-997b-9e16d27048e0/state","method":"PUT","req_id":"81fe7bc3-9ed4-45db-bd4c-76ff6ed38d84","remote_addr":"[fd00:1122:3344:101::a]:54567","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":945,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-07-26T02:45:37.071330661Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.071352041Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.071377542Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-26T02:45:37.071330661Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-26T02:45:13.770859467Z }","propolis_id":"400431bd-e96b-4ea7-9e2f-5a4d556819e4"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.071397162Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-26T02:45:13.770859467Z }, propolis_id: 400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-26T02:45:37.071330661Z }, migration_state: None }","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-07-26T02:45:37.136468681Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.136488962Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.136514602Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-07-26T02:45:37.136468681Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-26T02:45:37.136468681Z }","propolis_id":"400431bd-e96b-4ea7-9e2f-5a4d556819e4"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.136538123Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:621","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.136546913Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.13729031Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.137463714Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:445","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/2ea70c3f-a29d-4c51-ad99-4cf32a6e81b1/debug/bundle/zone\", \"/pool/int/f09c5312-7869-4743-8a0d-863eb3bf0156/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/4b40b5f5-772a-4d5a-bbcd-d2fa50a0ded8/crypt/debug\", \"/pool/ext/b8ef9471-5b81-40ed-90b5-1103a4a9eb91/crypt/debug\", \"/pool/ext/a47af5d0-e565-4513-84ad-c553b7265a72/crypt/debug\", \"/pool/ext/2a33f7b1-2bfd-4239-9138-fb534f4ccb3f/crypt/debug\", \"/pool/ext/a38a1ea7-44a1-411f-bbb8-767c9a23135f/crypt/debug\", \"/pool/ext/cb102b97-0352-4c03-8d37-4c80c4475510/crypt/debug\", \"/pool/ext/c5566b16-9a9f-498b-a4b1-83ac43308db5/crypt/debug\", \"/pool/ext/136ed548-ab63-44da-a187-0fa13dc111a0/crypt/debug\", \"/pool/ext/3ed9ecd9-4aeb-4ec2-8876-b9c8ed7df016/crypt/debug\"] }","zone_name":"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.175429391Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.176740942Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e0a246c7-df53-4162-a461-dc008dfe2796","remote_addr":"[fd00:1122:3344:101::a]:33729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":706,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.189898489Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"100d6fd9-bb6a-4126-953a-f851d2d6071e","remote_addr":"[fd00:1122:3344:101::a]:33729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":138,"response_code":"204"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:37.788587549Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1246","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4\", bundle_id: 9ea10160-1bed-4ef7-badb-c0082579c5e1 }, time_created: 2024-07-26T02:45:37.137860644Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_400431bd-e96b-4ea7-9e2f-5a4d556819e4","v":0,"name":"SledAgent","level":40,"time":"2024-07-26T02:45:37.788672171Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:793"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.541737428Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.556567754Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(9883005), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.556610145Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(9883005), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.556985234Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-26T02:45:37.136468681Z }, propolis_id: 400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-26T02:45:39.556958193Z }, migration_state: None }","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.647959818Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.648317177Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"c1252adc-9db2-4eb9-ae6b-ae889f4fbf3e","remote_addr":"[fd00:1122:3344:101::a]:41751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":271,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.648578563Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9883005) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.648613624Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"40f3ee68-f360-4992-a62c-a7a460a05c1a","remote_addr":"[fd00:1122:3344:101::a]:41751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.829068368Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-26T02:45:37.136468681Z }, propolis_id: 400431bd-e96b-4ea7-9e2f-5a4d556819e4 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-26T02:45:39.556958193Z }, migration_state: None }","instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.832626831Z","hostname":"sock","pid":1555,"instance_id":"4339d003-d6c5-45d3-997b-9e16d27048e0","component":"InstanceManager","file":"sled-agent/src/instance.rs:736"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.872753918Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:41172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.873505215Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cb0848c2-0f93-48c5-b4be-5a82ecc53be7","remote_addr":"[fd00:1122:3344:101::a]:41172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":717,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.879191248Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9efdf556-1e24-44ea-8609-88c90cb929d3","remote_addr":"[fd00:1122:3344:101::a]:41172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":106,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.917905822Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.918000095Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a476f4c0-4142-4bdd-95cc-a96601dfbff3","remote_addr":"[fd00:1122:3344:101::a]:37300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:39.925282734Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ee90ee6e-546a-49da-a5d1-a5203a29661d","remote_addr":"[fd00:1122:3344:101::a]:37300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":613,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:40.082454305Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47924"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:40.082652699Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"bfa70343-7bfa-4352-b40d-1ee1c3210ae4","remote_addr":"[fd00:1122:3344:101::a]:47924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":136,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:40.082965867Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:820","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 166, 16])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9883005) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:40.083003197Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"8de5140a-ac4d-4cc3-9fb5-342f2fee2ad8","remote_addr":"[fd00:1122:3344:101::a]:47924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":91,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:40.949511891Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65393"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:40.949641135Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f8f3a9ef-d01f-40d2-82ad-955914f024c2","remote_addr":"[fd00:1122:3344:101::c]:65393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":89,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:40.95802739Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"a13c8226-d2f7-463e-8fa6-295d70b69168","remote_addr":"[fd00:1122:3344:101::c]:65393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1167,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.004847484Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.005953429Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"9fbe8732-9e54-4175-ba53-2c7507e3a952","remote_addr":"[fd00:1122:3344:101::c]:45288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1081,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.136548909Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38723"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.13661056Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cab047c8-8c5d-4b70-8e89-490d44374959","remote_addr":"[fd00:1122:3344:101::a]:38723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.142542749Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"1e5e85f4-0d44-4522-9439-1673dcd8f04b","remote_addr":"[fd00:1122:3344:101::a]:38723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.145042937Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.145136869Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ca04193f-10ec-4c47-a3a0-ca7c994fc03a","remote_addr":"[fd00:1122:3344:101::b]:39735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":22,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.152586633Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e8a5e042-d00d-4bd9-9253-0ba08444cbce","remote_addr":"[fd00:1122:3344:101::b]:39735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.178249432Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50535"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.178396426Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.205637282Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.228460035Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ca527727-3fd9-4264-9fcb-a2fea8543f7f","remote_addr":"[fd00:1122:3344:101::c]:50535","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50116,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.228582858Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52717"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.229393566Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2856ebd0-bba2-4a02-9522-1cf8a1a9e706","remote_addr":"[fd00:1122:3344:101::b]:52717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":760,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.229425457Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"32034632-52ac-49ac-819c-54213decea09","remote_addr":"[fd00:1122:3344:101::c]:50535","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":721,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.325371088Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42530"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.325524811Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.350456093Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.37255914Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"661a9133-4779-4c73-8fcd-22f9e16c87f3","remote_addr":"[fd00:1122:3344:101::b]:42530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.372943589Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f6e3c3fa-ebae-43bb-a418-79cfcb9bf17f","remote_addr":"[fd00:1122:3344:101::b]:42530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.882245632Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38263"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:41.882446956Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"194edc87-089b-474c-a5d2-08d5ed6dad69","remote_addr":"[fd00:1122:3344:101::a]:38263","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:42.010975428Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:42.01106793Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ddccc7e1-b90e-4906-957e-b861f49c1e7d","remote_addr":"[fd00:1122:3344:101::a]:64060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:42.01961044Z","hostname":"sock","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"afb160be-f7a5-432e-9f38-4092c23ff430","remote_addr":"[fd00:1122:3344:101::a]:64060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:42.271079621Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:65496"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:42.271162843Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3820"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:42.295148843Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:42.31686792Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1b63c384-1cdf-4fff-80a3-84ed52e76f7a","remote_addr":"[fd00:1122:3344:101::a]:65496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45753,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-26T02:45:42.317123186Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"888eab76-6107-4697-9f48-6978fb48bb3d","remote_addr":"[fd00:1122:3344:101::a]:65496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":90,"response_code":"200"}