[ Oct 31 00:12:23 Enabled. ] [ Oct 31 00:12:23 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Oct 31 00:12:23 Method "start" exited with status 0. ] [ Oct 31 00:12:23 Rereading configuration. ] [ Oct 31 00:12:23 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:23.957570694Z","hostname":"sock","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:23.957750476Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.009621947Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.009660898Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.123405863Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.132476486Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.140382677Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.742532414Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.882603169Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.882628749Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.88272283Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.88278197Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.882813871Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.882824391Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.886001183Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:24.886012014Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.886028054Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.886037614Z","hostname":"sock","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.886055334Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.886134195Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.886205136Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.886229016Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.886251876Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.89737133Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.89739586Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:24.994643266Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.401654535Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.401701245Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.401709485Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.401722496Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.401733096Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.403534134Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.403598515Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.410453835Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.410459995Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.499724369Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.9059849Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.90602353Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906060581Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906067791Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906075701Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906081801Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906119962Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906125522Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906138212Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906141782Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906146002Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906161172Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906165912Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906172332Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906187482Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906191622Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906195062Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906199152Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906203923Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906219072Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906223503Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906231233Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906252223Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906255653Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906259583Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906263373Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906267353Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906271893Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906287753Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906292083Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906295823Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906299273Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906321884Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906326194Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906330064Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906335904Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906339274Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906343454Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906367104Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906370754Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906397074Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906404974Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906453455Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906464705Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906469145Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906482675Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906489455Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906522876Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906528186Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906531926Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906535966Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906539576Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906549376Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906553256Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906587026Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906592927Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906596356Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906599947Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906626427Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906630977Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906634177Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906662847Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906677387Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906685507Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906695288Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906702308Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906706277Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906731908Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.906737868Z","hostname":"sock","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.917286426Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:733"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.917312066Z","hostname":"sock","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999946979s","next_cleanup":"Instant { tv_sec: 1401, tv_nsec: 854183881 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.917355827Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.917362867Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3909"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.917404557Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.917421997Z","hostname":"sock","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.920407288Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922079675Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.92258896Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1074"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.92259569Z","hostname":"sock","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922658011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922703901Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922716472Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922722942Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922748432Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922756822Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922763682Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922770972Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1179"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922774332Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922777862Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922807853Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.922838133Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.923149726Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 91960f7f-96b0-4f89-964d-edd8b80ff639,\n sleds: {\n [fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: f95bcc91-ac8f-446a-8a84-3ca7f7459fd7 (sled),\n rack_id: 91960f7f-96b0-4f89-964d-edd8b80ff639,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.923419129Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.923424029Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1213"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.923460589Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.923471169Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.923622521Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.923757512Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.923808113Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: f95bcc91-ac8f-446a-8a84-3ca7f7459fd7 (sled), rack_id: 91960f7f-96b0-4f89-964d-edd8b80ff639, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.924439549Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4206"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.92450652Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4588"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.929197888Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.935016878Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.93522143Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.939835937Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.939889568Z","hostname":"sock","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.939986938Z","hostname":"sock","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:59491","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: f95bcc91-ac8f-446a-8a84-3ca7f7459fd7 (sled), rack_id: 91960f7f-96b0-4f89-964d-edd8b80ff639, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.94010085Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.94011726Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.94014949Z","hostname":"sock","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.94015891Z","hostname":"sock","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.9401674Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.94018636Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.940219701Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.940276912Z","hostname":"sock","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.940290781Z","hostname":"sock","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.940325072Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.940332912Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.940342762Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.940370462Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.940381353Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.95769184Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.979810206Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.990281773Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:401"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.990323064Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.990331134Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:413"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.990343174Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 112.224006ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.993022742Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a24f8b7-2dde-4cd6-83ac-a58e5cc8e1a0\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:12:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6a24f8b7-2dde-4cd6-83ac-a58e5cc8e1a0\" }"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:25.997306876Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"attempting to create encrypted zvol: name=\"oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.002857052Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.026120811Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.026169561Z","hostname":"sock","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.031256773Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:428"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.031266703Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.031270923Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.105974209Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 106.278469ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.106471744Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0d36a10-acd3-449d-8a7f-f13310b5aa32\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:12:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d0d36a10-acd3-449d-8a7f-f13310b5aa32\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.11685869Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.214035375Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 173.499055ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.214595131Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"25b37a77-573c-4426-a396-179b14560521\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:12:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"25b37a77-573c-4426-a396-179b14560521\" }"} {"msg":"Mounting oxi_39b08bb4-95dc-4ffe-8006-7954c81e10b8/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.335880163Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.348183439Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.34829862Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.348328471Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.348350451Z","hostname":"sock","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.389312371Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 232.323812ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.389936317Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b040bb8b-c7b6-412f-b864-91a16532603a\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:12:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b040bb8b-c7b6-412f-b864-91a16532603a\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.62256138Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 255.245271ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.623262957Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2417123-1f7a-487e-ab33-877e7d2db035\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:12:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2417123-1f7a-487e-ab33-877e7d2db035\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.88039322Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 155.981284ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:26.881055987Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"33867e01-3e22-4c2e-9c3f-d3eb3fecce07\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:12:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"33867e01-3e22-4c2e-9c3f-d3eb3fecce07\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.038437188Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.07569273Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.081235067Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:40997","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.081275497Z","hostname":"sock","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.081344328Z","hostname":"sock","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:40997","producer_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:27.082398079Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"152.407492ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.0834924Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.08349978Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.08354445Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:917","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:27.083550021Z","hostname":"sock","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:27.237044683Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"589.538382ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:27.828768743Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.04664005s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:28.876714656Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.796523639s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.598355278Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2664"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.598388088Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2882"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.598396778Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2534"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.598403158Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2581"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.598419048Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2963"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.598431748Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.600379258Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2807"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.601950894Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2575"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.602084136Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:29.602208467Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 32.407876ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:29.874970611Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 133.257829ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:30.09071967Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 222.570588ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:30.294415127Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 473.258107ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:30.689806686Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.057553924s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:31.331587149Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:31.674854935Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.765867686s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 823.371253ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:32.53353345Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.399699892s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:33.368053697Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 637.656992ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:34.843099284Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.794288846Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.794331177Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.794342537Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.794362697Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.7945876Z","hostname":"sock","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.794784642Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.794831102Z","hostname":"sock","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.794839702Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3909"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:35.794892712Z","hostname":"sock","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:59491","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.010614782Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.010700413Z","hostname":"sock","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.010862644Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.010887095Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.010910295Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.010948685Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.010978465Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.010994965Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.011060016Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.016177769Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4230","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.016189559Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4246"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.016194979Z","hostname":"sock","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.248609929Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:65074"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.248828941Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.248834692Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.248872212Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.248898292Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.249018173Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"77c78f72-a23c-4c26-972f-6451972d66ac","remote_addr":"[fd00:1122:3344:101::1]:65074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":276,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.278809528Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:39487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.279100211Z","hostname":"sock","pid":1572,"uri":"/sled-role","method":"GET","req_id":"80c63125-c506-4902-aba7-0f092f46b4de","remote_addr":"[fd00:1122:3344:101::1]:39487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":63,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.280661958Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.280693358Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.280987981Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.281209923Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:886"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 6617d11f-d09d-4ac1-adc2-df58ad02c088, pool_id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 45278ea7-29d3-4d40-9c55-4aa8089d331e, pool_id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 4b77586d-64ae-48a3-96bf-a7ac0c68508e, pool_id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: b8667009-7db6-4f53-9319-06981954ebcb, pool_id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: f39e3216-44dc-4792-9bc3-55d1e76cb852, pool_id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: fac52a67-aa6f-41bb-bfc5-e839c710a2fb, pool_id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 69c3e515-8df9-4d41-8e83-a7a2ff08777a, pool_id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 22e3722e-0394-4395-a899-668661ff1aa4, pool_id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 75ec6d24-5102-4d8c-b644-3d2f3b9572bb, pool_id: b43e5889-7a07-42ec-9599-254e55788daa (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.312636445Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:376"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.312746056Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:42737"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.312957178Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:874"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 6617d11f-d09d-4ac1-adc2-df58ad02c088, pool_id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 45278ea7-29d3-4d40-9c55-4aa8089d331e, pool_id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 4b77586d-64ae-48a3-96bf-a7ac0c68508e, pool_id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: b8667009-7db6-4f53-9319-06981954ebcb, pool_id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: f39e3216-44dc-4792-9bc3-55d1e76cb852, pool_id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: fac52a67-aa6f-41bb-bfc5-e839c710a2fb, pool_id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 69c3e515-8df9-4d41-8e83-a7a2ff08777a, pool_id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 22e3722e-0394-4395-a899-668661ff1aa4, pool_id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 75ec6d24-5102-4d8c-b644-3d2f3b9572bb, pool_id: b43e5889-7a07-42ec-9599-254e55788daa (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.313008599Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.313017209Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.313025709Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.313032909Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.313039519Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.313046519Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.313054259Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.313060399Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.468503111Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:36.468544732Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:36.94190443Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:37.159710431Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8943919027585428539"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:37.159844242Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7027826344887932491"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.374020066Z","hostname":"sock","pid":1572,"component":"VmmReservoirManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.405558179Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.413187317Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.413282298Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.413292838Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.413309858Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.413324908Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.41345384Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.41346483Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:37.443270805Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"8.09075212s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.448956653Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1d6671a7-5622-4671-b7bf-0a757330cf6b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.457292729Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_1d6671a7-5622-4671-b7bf-0a757330cf6b","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.631496653Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.639466505Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.639510505Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.639525955Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.639531025Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.639535105Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.650028173Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.650051313Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7dd5e852-297b-4341-9eef-72d5112891d5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.739249617Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.746019456Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.746049756Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.746058546Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.746078436Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.746089107Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.746217088Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.746227228Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.780073345Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7dd5e852-297b-4341-9eef-72d5112891d5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.787043056Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:37.944088935Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_7dd5e852-297b-4341-9eef-72d5112891d5","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.958026227Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.966417573Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.966481734Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.966490034Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.966496724Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.966502704Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.976403975Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:37.976430666Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.075366069Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.083514622Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.083580113Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.083589053Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.083623563Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.083639253Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.083797775Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.083807985Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.117585851Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_efb62c8a-6541-476c-8447-df5266c7fcb3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.125493392Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_efb62c8a-6541-476c-8447-df5266c7fcb3","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.298594975Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.306250473Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.306291744Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.306297814Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.306304074Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.306308114Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.315815782Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.315838802Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.409904615Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.417853937Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.417895887Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.417903627Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.417917617Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.417924967Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.418069569Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.418079589Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.453144858Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ca55523e-99d5-46dd-9043-66e4f721f4aa/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.460903797Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_ca55523e-99d5-46dd-9043-66e4f721f4aa","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.627889018Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.635399825Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.635432545Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.635441875Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.635448775Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.635467176Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.644322436Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.644349487Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.738512301Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.746106029Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.746146379Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.74619549Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.7462432Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.74625865Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.746408122Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.746415642Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.779723383Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_67e5554d-50d7-4b6f-b9c3-723d8231ab86/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.787371151Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:38.946203858Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_67e5554d-50d7-4b6f-b9c3-723d8231ab86","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.958732516Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.966447195Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.966488516Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.966530816Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.966541527Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.966546356Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.976442178Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:38.976469188Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.069493941Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.07719231Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.07722086Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.07724302Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.07724998Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.07726021Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.077422052Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.077445502Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.111837455Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.119915427Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.293799078Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.301527497Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.301556568Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.301570478Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.301576098Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.301604828Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.310696461Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.310729042Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.409363692Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.417087501Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.417119771Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.417131371Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.417170642Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.417181242Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.417300913Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.417309873Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.451980138Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a2173f6b-5437-4ef6-a016-ab4b5f89f5d8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.459096021Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_a2173f6b-5437-4ef6-a016-ab4b5f89f5d8","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.624235583Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.63179227Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.63183057Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.631846981Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.631853541Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.631857421Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.640765772Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.640786922Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.736317401Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.74311859Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.743143961Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.74315007Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.743167921Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.743176351Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.743341632Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.743360443Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.778946227Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a536fa1e-3243-42d2-9446-aa31a45af0ee/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.78697817Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:39.948413113Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_a536fa1e-3243-42d2-9446-aa31a45af0ee","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.967586979Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.975621071Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.975654012Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.975689182Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.975697302Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.975701632Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.985786526Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:39.985819116Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b43e5889-7a07-42ec-9599-254e55788daa (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.086082263Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.094073685Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.094141525Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.094150905Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.094179226Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.094191076Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.094319977Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.094327787Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.129849841Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b43e5889-7a07-42ec-9599-254e55788daa/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.137974454Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_b43e5889-7a07-42ec-9599-254e55788daa","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.315747805Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.32407873Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324119001Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324196732Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324304373Z","hostname":"sock","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 39b08bb4-95dc-4ffe-8006-7954c81e10b8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3ddddc72-3f09-485d-a0b7-57e5a0e73bb2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324362353Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324370103Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324515485Z","hostname":"sock","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324749897Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:878"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324772838Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324785968Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:904","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324792418Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:909"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324799218Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:914"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324814428Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324820728Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:920"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324868468Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324877139Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:925"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.324956369Z","hostname":"sock","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"d4f3cf3d-c781-4758-ac20-448471935974","remote_addr":"[fd00:1122:3344:101::1]:42737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4012134,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.35625814Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.356351341Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:55488"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.356577133Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.356585963Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.407538015Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.407575495Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.457453426Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.457488387Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.50659798Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.50663177Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.50663741Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.506678741Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.550152956Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.550188996Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.550195386Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.601230299Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.605063548Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.608592534Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.621219304Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.625859221Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:40.629470628Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:40.950624157Z","hostname":"sock","pid":1572,"component":"ProbeManager","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:42.153996602Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:42.154095083Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:42.630974147Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:42.631036878Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:43.112159196Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:43.112294147Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:43.112304307Z","hostname":"sock","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:43.112312177Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 41.290383ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.126413482Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:43.126457902Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 147.573602ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.177573576Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf\") failed: Property not found. retry in 44.50627ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.248849876Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac\") failed: Property not found. retry in 63.641545ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.254908678Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf\") failed: Property not found. retry in 68.297955ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.300169721Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac\") failed: Property not found. retry in 103.188109ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.389097222Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf\") failed: Property not found. retry in 189.776748ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.410117278Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 176.142148ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.425779308Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac\") failed: Property not found. retry in 209.555142ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.522326827Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf\") failed: Property not found. retry in 485.965918ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.615800044Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 201.058046ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.639490797Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac\") failed: Property not found. retry in 228.147035ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.745194669Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 607.010979ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.850911842Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac\") failed: Property not found. retry in 995.328547ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:43.982486649Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf\") failed: Property not found. retry in 969.440851ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:44.110372469Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 707.053584ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:44.468048753Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac\") failed: Property not found. retry in 647.813617ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:44.988886827Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf\") failed: Property not found. retry in 798.103859ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:45.093437228Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 1.272723009s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:45.184299978Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac\") failed: Property not found. retry in 1.138714892s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:45.645775885Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:45.797001214Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3558162335736431262"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:45.797128865Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10693765855761231549"} {"msg":"wait for service Some(\"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf\") failed: Property not found. retry in 950.814857ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:45.90006974Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:45.907079881Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4254"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:45.907088552Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4264","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:45.91282157Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4343"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.014001916Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4391"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.014041047Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4418"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.053813794Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4431"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.053840884Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4295"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.107545114Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4337"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.107574485Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4398","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.127129055Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4409"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.127160826Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4451"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.194392094Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4483"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.194429164Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4489"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.282167803Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4530"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:46.282218174Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7)","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:46.315392543Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Failed to get switch ID from MGS (retrying in 31.056067ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:46.315492735Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 102.751123ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:46.347807105Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 254.901734ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:46.451949102Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 1.015631829s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:46.464992726Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 470.502072ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:46.708288967Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"wait for service Some(\"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac\") failed: Property not found. retry in 1.395533638s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:46.793533671Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf\") failed: Property not found. retry in 1.493017033s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:46.85889623Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.098391133Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:35442"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.098659765Z","hostname":"sock","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"35ce6436-817f-4315-a0b0-d105411bc26b","remote_addr":"[fd00:1122:3344:101::2]:35442","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:47.162943344Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8943919027585428539"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:47.163146486Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7027826344887932491"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.180532104Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.211799194Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.211902205Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:724"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.212221659Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:727","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.212241309Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.249090036Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293295369Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29337239Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29338065Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29338915Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29339565Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29340217Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29340849Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29341638Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29342267Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29342908Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29343563Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.29344358Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293449731Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293456051Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293473661Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293478551Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293483111Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293487571Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293492131Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293496631Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293501491Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293506001Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:47.293510291Z","hostname":"sock","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17\") failed: Property not found. retry in 1.486159689s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:47.490050594Z","hostname":"sock","pid":1572,"zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:48.995149329Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_2601250f-fbc7-4c23-8a05-171c621b81cf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:48.99519811Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_a89ae16b-d375-45df-91ec-655f19545f17"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:48.99521004Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_750c1512-9219-4fef-8dd4-e9a22f8ae5ac"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:48.99522025Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:48.995540193Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:48.995796326Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"7958c44e-f55c-4b36-b5aa-4896457589d4","remote_addr":"[fd00:1122:3344:101::1]:55488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8639371,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:48.997462073Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.046408394Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.093535117Z","hostname":"sock","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.138934942Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.138949852Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.169414884Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.170255973Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.170261173Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.19933476Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.199408451Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:54328"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.199593143Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.199602763Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.21981276Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.219855531Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.238964866Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.239020237Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.255607627Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.255677388Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.255867239Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.256039381Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.266851222Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.298746449Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.299304484Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.299313435Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.329858257Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 214, 36]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.329875848Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.33016744Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.330180061Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.330399593Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.330780027Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.353105355Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.370646065Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.993532955Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:49.993664126Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 33.821416ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:50.125537917Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 76.566629ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:50.166055922Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 248.779017ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:50.261159316Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 217.398235ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:50.519949737Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 701.683378ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:50.747196354Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 740.410465ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:51.459655681Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 520.74791ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:52.209124537Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 1.302574104s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:52.739324877Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f\") failed: Property not found. retry in 1.444331711s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:54.052198683Z","hostname":"sock","pid":1572,"zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.51640983Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.516474461Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.516843435Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.517159508Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b4129e32-e289-45d3-9cf2-669d26c554fe","remote_addr":"[fd00:1122:3344:101::1]:54328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6317691,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.519534902Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:653"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.548968994Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.549137446Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:61456"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:55.557061117Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.557183938Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"4d6b5789-e775-4931-84ce-6e7f079ba61e","remote_addr":"[fd00:1122:3344:101::1]:61456","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7889,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:55.56035518Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.683211229Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:55.68334543Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:54538"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:55.798930824Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3558162335736431262"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:55.799057365Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10693765855761231549"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:57.164598251Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8943919027585428539"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:57.164722302Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7027826344887932491"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:58.21604916Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11284587146137120395"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:58.376430822Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4145109922441483943"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:58.997814796Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"16779317648949940276"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:59.69036573Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:59.690501371Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"636cb7b8-0cb5-4210-9ba2-3af92ad5f894","remote_addr":"[fd00:1122:3344:101::1]:54538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007080,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:59.692861185Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:59.848088275Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:59.848231807Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:41549"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_14078d86-af25-4a11-bd50-933b6f13166f': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-31T00:12:59.857269139Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:12:59.85737488Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"e25627c8-061c-4a3d-ade7-961c0f0d20af","remote_addr":"[fd00:1122:3344:101::1]:41549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9081,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:12:59.859867866Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.083252654Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.083400095Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:61557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.092036704Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"d6f089ab-ac89-4951-837d-21bcd495e0cf","remote_addr":"[fd00:1122:3344:101::1]:61557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8581,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.093770392Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:00.093778681Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.24002419Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.240147011Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:47639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.249508767Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"35e5a94e-ff1f-40fe-ab65-e2473fa33165","remote_addr":"[fd00:1122:3344:101::1]:47639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9310,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.251018152Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:00.251024222Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.5016999Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.501867821Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:36004"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.510043825Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"d21cb93a-beee-4491-8d24-4bc54102ef51","remote_addr":"[fd00:1122:3344:101::1]:36004","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8092,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.513137867Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:00.513145577Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.731074489Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.73120306Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51383"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.738733467Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"a0d626f9-dc93-40f5-9187-0ad1f49e10b4","remote_addr":"[fd00:1122:3344:101::1]:51383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7481,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.740562916Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:00.740568726Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.945798338Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.945887379Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:33658"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.953391256Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"6dfda824-053f-44c7-9e28-385ad2564c78","remote_addr":"[fd00:1122:3344:101::1]:33658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7389,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:00.956889672Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:00.956898192Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.198046811Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.198156423Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:49980"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.205874842Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"95218e3b-f519-48c6-a56b-9d2099c5503c","remote_addr":"[fd00:1122:3344:101::1]:49980","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7576,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.208467618Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.208472978Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.033844627Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.034005248Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:65286"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.054516579Z","hostname":"sock","pid":1572,"uri":"/timesync","method":"GET","req_id":"cc880ceb-087a-4616-a140-cb519048f3cc","remote_addr":"[fd00:1122:3344:101::1]:65286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":20406,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 759117325, ip_addr: 45.63.54.13, stratum: 3, ref_time: 1730333580.7595258, correction: -0.048131339 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.058810583Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.058815503Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1299"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.08877967Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.088893341Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:33601"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.089173924Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.089180194Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.108473541Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.108508082Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.126142542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.126176623Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.145209307Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.145242238Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.194985087Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.195018768Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.243307082Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.243341093Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.292330845Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.292363175Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.341022963Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.341058913Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.39534386Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.39538112Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.395564632Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.395739344Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.425853382Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.425881192Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.425887622Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.425894692Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.425902672Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.426006253Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"508b234d-9acc-4597-b58e-4ba4530761b8","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized)]","latency_us":337036,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.426198415Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"116.963977ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"508b234d-9acc-4597-b58e-4ba4530761b8\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"508b234d-9acc-4597-b58e-4ba4530761b8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.544432446Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.544957452Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.544975822Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.563210599Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.563234019Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.580084772Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.580109132Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.59755096Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.597579561Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.614901268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.614927828Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.632993673Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.633030054Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.64931251Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.649341891Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.667571088Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.667595268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.685007126Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.685049396Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.685241229Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.68539882Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.716555019Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.71659597Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.71660275Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.71662289Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.71663064Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.716723081Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a4e9d1b0-8b0b-4393-aa94-dfedef6b180c","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized)]","latency_us":172003,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.716894483Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"72.888596ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a4e9d1b0-8b0b-4393-aa94-dfedef6b180c\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a4e9d1b0-8b0b-4393-aa94-dfedef6b180c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.791245364Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.791556527Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.791568638Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.810580442Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.810609753Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.827718778Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.827753268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.846048525Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.846084696Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.864566505Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.864595026Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.882390018Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.882415258Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.900642735Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.900680995Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.917971182Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.918015233Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.936023877Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.936059717Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.93628884Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.936423271Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.968890454Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.968930984Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.968940614Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.968949754Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.968957634Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:01.969087276Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"25fe81dd-7b1f-4f9e-96e9-cdee23d3f23c","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized)]","latency_us":177707,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:01.969266267Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"104.87395ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"25fe81dd-7b1f-4f9e-96e9-cdee23d3f23c\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"25fe81dd-7b1f-4f9e-96e9-cdee23d3f23c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.075229035Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.075566468Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.075578809Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.094607314Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.094654474Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.111588588Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.111628909Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.128687011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.128732424Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.145923771Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.145966101Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.163864325Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.163912836Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.181085792Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.181118782Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.198182117Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.198219938Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.216024091Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.216071651Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.216266093Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.216405295Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.247518974Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.247556925Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.247563295Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.247570815Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.247578205Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.247702216Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"3bc128d0-3ec9-48a1-af94-66370aa1088b","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized)]","latency_us":172783,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.247821557Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"229.479744ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3bc128d0-3ec9-48a1-af94-66370aa1088b\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3bc128d0-3ec9-48a1-af94-66370aa1088b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.477553737Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.477759329Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.477771069Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.496235848Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.496266679Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.5148948Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.51493077Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.532917635Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.532947266Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.55094556Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.550984671Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.569584112Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.569611322Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.587542156Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.587577407Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.605799044Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.605826544Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.624115542Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.624149392Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.624363434Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.624493876Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.656632536Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.656673306Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.656682786Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.656692146Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.656701646Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.656799407Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"28029f0f-3d82-4d96-8bdf-5020d0a5230d","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized)]","latency_us":179621,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:02.656958159Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"256.978253ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"28029f0f-3d82-4d96-8bdf-5020d0a5230d\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"28029f0f-3d82-4d96-8bdf-5020d0a5230d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.914660966Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.91505065Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.91506593Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.936084046Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.936118566Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.953833378Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.953866368Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.969435918Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.969468478Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.986842637Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:02.986875817Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.003827011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.003866332Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.021173949Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.02122179Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.039177974Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.039210185Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.056821906Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.056893246Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.057109669Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.057326331Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.088746594Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.088794004Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.088803084Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.088821024Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.088827794Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.088907165Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"696a19ff-b5b6-4e2f-85a1-db030f514c74","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized)]","latency_us":174553,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.089086967Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"297.349131ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"696a19ff-b5b6-4e2f-85a1-db030f514c74\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"696a19ff-b5b6-4e2f-85a1-db030f514c74\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.386697174Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.386988077Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.386999277Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.407205914Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.407246274Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.425021737Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.425059067Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.442319714Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.442355025Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.45943693Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.459460581Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.478030642Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.478063622Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.49638538Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.49641814Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.514956951Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.514994071Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.533141877Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.533181738Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.53337837Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.533504881Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.564975484Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.565015014Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.565025035Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.565034625Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.565042335Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.565171526Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ff5e51e8-4363-4ad5-b885-b17557f93512","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized)]","latency_us":178836,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:03.565351698Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"309.748907ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ff5e51e8-4363-4ad5-b885-b17557f93512\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ff5e51e8-4363-4ad5-b885-b17557f93512\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.874856427Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.875278861Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.875354342Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.894445658Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.894492498Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.912938428Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.912983038Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.930588069Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.930635929Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.948997318Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.949039998Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.967143475Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.967178275Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.984720885Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:03.984769145Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.002660939Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.00269735Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.020399311Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.020463792Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.020681644Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.020875806Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.052074817Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.052107427Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.052113767Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.052121227Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.052129037Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.052259609Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"549f77f1-ef36-4b5b-a794-57e450029dea","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized)]","latency_us":177684,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.052462542Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"218.139011ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"549f77f1-ef36-4b5b-a794-57e450029dea\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"549f77f1-ef36-4b5b-a794-57e450029dea\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.271306948Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.271643652Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.271677702Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.292083372Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.292110882Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.309340309Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.309373699Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.326792918Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.326821788Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.344956335Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.344999395Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.364888729Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.36492753Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.383193927Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.383220168Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.401042011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.401072721Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.419148806Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.419178547Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.419379219Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.41951556Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.450811012Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.450861512Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.450870852Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.450877972Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.450887243Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.451000784Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"8293b467-e7bd-4fa0-bd09-1022022b0138","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized)]","latency_us":180034,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:04.451154615Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"396.171428ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8293b467-e7bd-4fa0-bd09-1022022b0138\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8293b467-e7bd-4fa0-bd09-1022022b0138\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.847513926Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.8478778Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.84789231Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.867730493Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.867784634Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.885683338Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.885728438Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.90341873Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.90346942Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.920551086Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.920592586Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.938520011Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.938571171Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.956241253Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.956265413Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.974614931Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.974647351Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.993496155Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.993533996Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.993717217Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:04.993837228Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.012598911Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"369.866487ms"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.027392113Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.027432174Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.027455114Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.027462014Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.027470134Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.027567105Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9b15d538-eb7f-4f77-a1dd-3dfa41494135","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized)]","latency_us":180397,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.027763837Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"741.360785ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9b15d538-eb7f-4f77-a1dd-3dfa41494135\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9b15d538-eb7f-4f77-a1dd-3dfa41494135\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.767564965Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.767863668Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.767877108Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.786782312Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.786835703Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.804776427Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.804807697Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.822656982Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.822696401Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.842005349Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.842036419Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.859924613Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.859957003Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.877245951Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.877293972Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.896024434Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.896061574Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.913895357Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.913937158Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.914104159Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.914240811Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.945706064Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.945773945Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.945785395Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.945793825Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.945801805Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:05.945873416Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"82c67722-582a-41a2-a150-baad949c31f0","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized), (\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized)]","latency_us":178671,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:05.946012127Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"590.303585ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"82c67722-582a-41a2-a150-baad949c31f0\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"82c67722-582a-41a2-a150-baad949c31f0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.535901775Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.536268999Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.536286509Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.555288614Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.555334825Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.572878665Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.572920115Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.592044202Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.592090812Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.610893145Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.610937446Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.629058522Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.629094822Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.647903586Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.647941796Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.665783949Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.66583182Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.683792414Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.683851765Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.684016606Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.684173188Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:06.715294908Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:06.715337018Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:06.715343358Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:06.715349288Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:06.715354878Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:06.715468629Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"6b0b9796-24d8-4f1c-a09b-6a57844c6740","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\", TimeNotSynchronized), (\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\", TimeNotSynchronized), (\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\", TimeNotSynchronized), (\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\", TimeNotSynchronized), (\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\", TimeNotSynchronized)]","latency_us":179904,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:06.715658801Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"658.081037ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b0b9796-24d8-4f1c-a09b-6a57844c6740\", \"content-length\": \"124\", \"date\": \"Thu, 31 Oct 2024 00:13:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b0b9796-24d8-4f1c-a09b-6a57844c6740\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.373229075Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.373599588Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.373612599Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.391816746Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.391873706Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.408784959Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.40881872Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.426462921Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.426503311Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.444295844Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.444330854Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.462176087Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.462216798Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.479247303Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.479294164Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.49609681Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.496142577Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.512934699Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.512977689Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.513229962Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.513399624Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.592147083Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.592195083Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.592202843Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.592210203Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.592217303Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.686594449Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.690814886Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.695047039Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.699345283Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.703549826Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.707896701Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.712211376Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.716427809Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.720753293Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:07.725835956Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.628532435Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.628672427Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.628688867Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.634032402Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.6679032Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.668079452Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.679792562Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.679959814Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.705702748Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:10.70586834Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 43.42209ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:10.77113466Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 34.948242ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:10.780568877Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 60.127323ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:10.804466522Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 36.585947ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:10.814499825Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 74.627288ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:10.820863451Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 149.316677ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:10.827919633Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 36.568801ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.004610395Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 143.527706ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.068891478Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 297.236143ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.137889877Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 121.930576ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.510331891Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 101.334099ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.566381507Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 84.597842ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.590842218Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 185.701814ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.628356633Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 234.138582ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.680478899Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 176.349352ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.715074354Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 230.346306ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.740791908Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 223.760477ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.770610364Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 534.77808ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.866521859Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 540.479083ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.923190031Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 509.702471ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:11.957597715Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 847.920757ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:12.019040376Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 412.791447ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:12.053347908Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 937.465388ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:12.437674585Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 821.170616ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:12.565395378Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 442.362989ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:12.678093544Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 692.463594ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:12.736383273Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.471994534s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:12.957980758Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.162927605s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:13.175770925Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 635.280391ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:13.395436401Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 719.69628ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:13.410549756Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 911.952368ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:13.436972757Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 923.094134ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:14.040241973Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 828.897591ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:14.137962536Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.022076588s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:14.344881852Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 523.684335ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:14.355923605Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 613.587ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:14.43439294Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 948.000721ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:14.887460543Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 988.544471ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:14.970154782Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 713.96363ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:14.977002563Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 804.816741ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:15.056009424Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.097058601s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:15.373883998Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:15.529478256Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37613"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:15.529686688Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"82824b63-3dfb-4644-b532-e9e22bfb5f21","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37613","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.182442415s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:15.698650453Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 548.761215ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:15.841784543Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 717.843358ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:15.86771933Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 1.057026619s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:15.965412523Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 596.235733ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:16.401918976Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 896.671423ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:16.480955188Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 626.702913ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:16.597556715Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 626.401326ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:16.890846477Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 1.492634928s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:17.00892753Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 647.574208ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:17.029860835Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.480586716s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:17.231188613Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 533.340458ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:17.383437966Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.15305792s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:17.524319773Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 1.495352204s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:17.683501164Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.418740025s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:17.923707145Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 1.068752458s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:18.507735592Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.00018289s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:18.685170345Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 903.31878ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:18.717355535Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 921.605487ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:19.184370291Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.05979003s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:19.348657088Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 626.363016ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:19.583390959Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.31686853s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:19.627028677Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.192556985s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:19.69157541Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 665.419241ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:20.113130537Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 780.412975ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:20.217058583Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:20.377417027Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"697.00719ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.070479075s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:20.415148654Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 532.472239ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:20.786846805Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.207059896s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:20.890397977Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.430733181s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:20.950510933Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 1.408485635s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:21.00580672Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 798.697377ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:21.328742556Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.381526725s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:21.494190929Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 987.874943ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:22.106032954Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 814.951189ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:22.136211673Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 831.438053ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:22.390249281Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 1.303158127s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:22.423930832Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 742.45015ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:22.885783003Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 533.302944ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:22.961332296Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.344403668s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:23.102808367Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.239709783s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:23.230734756Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 1.092834353s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:23.504238728Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 686.517317ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:23.638150609Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 1.431567739s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:23.737234784Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 734.562636ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:24.334093301Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.071300339s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:24.457264772Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.135539718s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:24.479869863Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 1.337956155s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:24.606632721Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 881.369237ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:25.078087459Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 1.401986116s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:25.178702049Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 554.231768ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:25.537321092Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.239966836s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:25.62402962Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 659.346355ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:25.955119222Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 733.100671ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:25.967852173Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 702.348301ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:26.101164908Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 1.000240526s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:26.588893847Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 1.467116707s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:26.621973006Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 904.13823ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:26.709104159Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.478066418s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:26.812352407Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.42024481s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:26.872731835Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 827.626919ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:27.597507723Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.341155272s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:27.621271366Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 1.204592974s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:28.097743129Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 1.414495839s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:28.297955031Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 918.253074ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:28.304977613Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 857.55077ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:28.432905903Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 1.00362862s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:28.970489281Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 723.124679ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:29.232198052Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 738.02967ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:29.298523831Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0\") failed: Property not found. retry in 1.392654421s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:29.31113756Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389\") failed: Property not found. retry in 907.821825ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:29.721164599Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb\") failed: Property not found. retry in 1.471576132s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:29.965733754Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae\") failed: Property not found. retry in 938.713485ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:29.985858681Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173\") failed: Property not found. retry in 1.406370689s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:13:30.049887158Z","hostname":"sock","pid":1572,"zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.480022786Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_24b8d53a-5865-43c7-bd1c-6c377ecc3aae"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.480062306Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_8f2b3ece-56e1-4bdd-9e3d-b87256e617b0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.480068236Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_3ff969c4-5131-4f29-bd5a-e8433819d173"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.480072196Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_8e581c4b-8e35-4d00-b402-3a7777b44389"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.480076266Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_f7bfd91e-056e-4254-b3ef-a9b7a528f9cb"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.480111027Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.48046891Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.480780524Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"533a8f26-af11-42d1-870f-cf9b39f9b7bc","remote_addr":"[fd00:1122:3344:101::1]:33601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":24140658,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.513036662Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34741"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:31.542275723Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3705"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:13:47.14542769Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3723"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.219390076Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.22269425Z","hostname":"sock","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"c17b84f1-d712-4b6b-8f55-13cc5431f215","remote_addr":"[fd00:1122:3344:101::1]:34741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30709252,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.224545699Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55643"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:02.227826102Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"983.498144ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 2601250f-fbc7-4c23-8a05-171c621b81cf (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a89ae16b-d375-45df-91ec-655f19545f17 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 14078d86-af25-4a11-bd50-933b6f13166f (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 25fe17f6-294a-4cf1-b7fb-da39dd364f5d, kind: Service { id: 14078d86-af25-4a11-bd50-933b6f13166f }, name: Name(\"ntp-14078d86-af25-4a11-bd50-933b6f13166f\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 36])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3ff969c4-5131-4f29-bd5a-e8433819d173 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e581c4b-8e35-4d00-b402-3a7777b44389 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name(\"external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name(\"external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name(\"nexus-1e4bfc38-893c-408c-9d22-948390ba9d86\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name(\"nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name(\"nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4ce79729-70af-4ea9-8907-92eccad979f8 (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 12eca9ce-450a-47c6-aa79-35164c1f047f (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8377dba6-b489-43b3-98e1-05bb7df98988 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 065751ee-697c-4a72-8cf7-687dac3ede9c (service), filesystem_pool: Some(ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 511547b5-244c-4c59-b939-8d95795a2c87 (service), filesystem_pool: Some(ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a4913a46-2b97-4d0c-965d-3da766243586 (service), filesystem_pool: Some(ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External } } } }, OmicronZoneConfig { id: e2b91b21-636b-4131-93b7-4d20d375703a (service), filesystem_pool: Some(ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ee7d195-9a59-4752-ad62-863aa8d03aa4 (service), filesystem_pool: Some(ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 32137949-0a38-4d1e-a737-2d07177c3738 (service), filesystem_pool: Some(ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f59615c-c60c-43bf-8c6b-f46c94caeddb (service), filesystem_pool: Some(ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1e02403f-00b9-457b-a823-4da05e840386 (service), filesystem_pool: Some(ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External } } } }, OmicronZoneConfig { id: d634556d-8226-4df6-8dfc-09c66049e16b (service), filesystem_pool: Some(ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: c559bdc6-d957-40c2-ad54-ae7947822ea7 (service), filesystem_pool: Some(ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External } } } }, OmicronZoneConfig { id: dde78775-b16f-45ca-91b2-9608c686b32c (service), filesystem_pool: Some(ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.261384406Z","hostname":"sock","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.261692479Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:44733"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.263512128Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.263522228Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2601250f-fbc7-4c23-8a05-171c621b81cf, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.290246692Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.290295682Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a89ae16b-d375-45df-91ec-655f19545f17, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.310473779Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.310522119Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 750c1512-9219-4fef-8dd4-e9a22f8ae5ac, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.331010279Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.33105473Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f7bfd91e-056e-4254-b3ef-a9b7a528f9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.349108484Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.349150465Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f2b3ece-56e1-4bdd-9e3d-b87256e617b0, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.371454563Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.371498284Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ff969c4-5131-4f29-bd5a-e8433819d173, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.392386367Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.392426058Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e581c4b-8e35-4d00-b402-3a7777b44389, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.411649845Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.411688675Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 24b8d53a-5865-43c7-bd1c-6c377ecc3aae, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.429465827Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.429503328Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2beb08a-ae3f-469f-86c4-36645336e7c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.55468268Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.5547377Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b2beb08a-ae3f-469f-86c4-36645336e7c7, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12eca9ce-450a-47c6-aa79-35164c1f047f, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.715665149Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.715722789Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 12eca9ce-450a-47c6-aa79-35164c1f047f, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a4913a46-2b97-4d0c-965d-3da766243586, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.825947098Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.825994289Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a4913a46-2b97-4d0c-965d-3da766243586, dataset_name: DatasetName { pool_name: ZpoolName { id: 1d6671a7-5622-4671-b7bf-0a757330cf6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2b91b21-636b-4131-93b7-4d20d375703a, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.965283185Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:02.965332986Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2b91b21-636b-4131-93b7-4d20d375703a, dataset_name: DatasetName { pool_name: ZpoolName { id: 7dd5e852-297b-4341-9eef-72d5112891d5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ee7d195-9a59-4752-ad62-863aa8d03aa4, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.105650443Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.105702433Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ee7d195-9a59-4752-ad62-863aa8d03aa4, dataset_name: DatasetName { pool_name: ZpoolName { id: efb62c8a-6541-476c-8447-df5266c7fcb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 32137949-0a38-4d1e-a737-2d07177c3738, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.237254731Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.237308531Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 32137949-0a38-4d1e-a737-2d07177c3738, dataset_name: DatasetName { pool_name: ZpoolName { id: ca55523e-99d5-46dd-9043-66e4f721f4aa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f59615c-c60c-43bf-8c6b-f46c94caeddb, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.351458941Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.351513441Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7f59615c-c60c-43bf-8c6b-f46c94caeddb, dataset_name: DatasetName { pool_name: ZpoolName { id: 67e5554d-50d7-4b6f-b9c3-723d8231ab86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1e02403f-00b9-457b-a823-4da05e840386, dataset_name: DatasetName { pool_name: ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.465297796Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.465343947Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1e02403f-00b9-457b-a823-4da05e840386, dataset_name: DatasetName { pool_name: ZpoolName { id: 1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d634556d-8226-4df6-8dfc-09c66049e16b, dataset_name: DatasetName { pool_name: ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.590475889Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.590524689Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d634556d-8226-4df6-8dfc-09c66049e16b, dataset_name: DatasetName { pool_name: ZpoolName { id: a2173f6b-5437-4ef6-a016-ab4b5f89f5d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c559bdc6-d957-40c2-ad54-ae7947822ea7, dataset_name: DatasetName { pool_name: ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.73024118Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.73029373Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c559bdc6-d957-40c2-ad54-ae7947822ea7, dataset_name: DatasetName { pool_name: ZpoolName { id: a536fa1e-3243-42d2-9446-aa31a45af0ee (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dde78775-b16f-45ca-91b2-9608c686b32c, dataset_name: DatasetName { pool_name: ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.871008602Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:03.871057162Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: dde78775-b16f-45ca-91b2-9608c686b32c, dataset_name: DatasetName { pool_name: ZpoolName { id: b43e5889-7a07-42ec-9599-254e55788daa (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.012986516Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.013140798Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.013713063Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.014027687Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.135142707Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.171301407Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.17154885Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.17158484Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.22234683Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.222552202Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.222564343Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.222571352Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.222588383Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.222599143Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.388622603Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.388808845Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.388881926Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.388915556Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.388930586Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.390068268Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.427864065Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.427900285Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.427905735Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.428469001Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.433987277Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.439604625Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.444607426Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.449903791Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.454625649Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.460084935Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.607419384Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.611830699Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.616252994Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.620408417Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.625004264Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.629149146Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.634303039Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.640710765Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.646687286Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.651677507Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.656752999Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.662538378Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.668083595Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.672724143Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.714095296Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.714256558Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.720011067Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.724130249Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.728600565Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.733885229Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.738404055Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.743069133Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.786299076Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.823375646Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.881856584Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.938490085Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.941322384Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.941337534Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.992446407Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 145, 100]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.992478697Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.993516418Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:04.993530168Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.048842615Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 239, 133]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.048889865Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.050807265Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.050823875Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.114684889Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 193, 148]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.114715629Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.116486488Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.116499528Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.180342162Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 244, 222]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.180374762Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.181966178Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.181978549Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.235275634Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 150, 109]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.235318805Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243333897Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243350157Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243418198Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243426318Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243506859Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243519889Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243572909Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.24358324Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243727811Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.243740221Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.251914475Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.253623842Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.29150738Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.295882455Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.296205878Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.299541893Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.299650574Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.302556844Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.302640404Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.4320732Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.481798219Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.497956505Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.516514415Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.551686945Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.576685861Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.609348695Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.621937124Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.640712787Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.658836842Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:05.679766707Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.177364067Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.177537679Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.186235638Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.192475792Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.199874638Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.208229983Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.936770225Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.936913836Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.944278902Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.944526074Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.944810927Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.944856908Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.951768608Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.959534798Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.966228087Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.966440249Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.966646141Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.973684483Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.980939317Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.98121566Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.981379622Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:08.987768057Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 50.418459ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.181254329Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 73.562693ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.194265082Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 55.857562ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.207849621Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 29.818995ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.221531341Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 31.551104ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.232739596Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 36.987592ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.242983951Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 39.753867ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.252939473Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 71.984506ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.264098347Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 142.202023ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.335805152Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 73.67873ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.476971087Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 70.310116ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.628012574Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:09.628801353Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 60.912733ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.692185832Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:09.692553846Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 103.745635ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.755618761Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 52.699225ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.793270957Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 137.775795ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.819800409Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 57.639006ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.85113911Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 61.235402ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.881861724Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 226.445292ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:09.95958748Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 73.526448ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.017563564Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 84.672161ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.091682943Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 56.409459ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.161276116Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 76.302432ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.24173326Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:10.242378037Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 137.744494ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.366031853Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 268.761407ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.41361245Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 124.904219ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.486892331Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 234.97103ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.561745928Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 255.359319ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.606474616Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 274.774832ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.679846247Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 357.498718ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:10.77771918Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 225.935697ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:13.79008692Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 203.271583ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:13.964255496Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 124.453297ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.175807512Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 229.533158ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.248645548Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:14.24875743Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 58.629543ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.312007647Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 127.082505ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.341284337Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 429.870256ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.354620664Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 287.406289ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.366087581Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 222.394847ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.377041913Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:14.379935793Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 524.456004ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.392900286Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 395.06363ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.403444734Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 597.051577ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.413669108Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 591.599097ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.423207736Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 498.075697ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.432776574Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 407.496529ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.442486474Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 99.668542ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.453429536Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:14.453601328Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 254.868933ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.480736055Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 398.893328ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.493489226Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 231.812042ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.56905139Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 53.961718ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.584446378Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 474.126007ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.615076341Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:14.615602907Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:14.61591974Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 52.148221ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.650994389Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 988.85805ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.665937562Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 70.201128ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.691635995Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 201.833578ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.715905434Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 682.663998ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.7506479Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 69.052768ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.772484583Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 1.033140241s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.799728092Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 594.883362ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.813468723Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 387.142889ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.831955433Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 114.246521ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.854213761Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 45.071986ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.865147853Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 697.339219ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.881794883Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 957.614216ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.907583857Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 115.578103ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.92343462Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 1.14856388s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.93913029Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 330.785578ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.959460909Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 756.692997ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.975507093Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 201.404526ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:14.993381526Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 711.279298ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.026729598Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 788.328618ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.043169086Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 177.566563ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.054753734Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 1.372776528s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.103570885Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.14028315s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.210965665Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 1.067592399s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.234466105Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 261.135565ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.248436988Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 522.691454ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.304561403Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 668.89078ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.424113847Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:15.428569173Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:15.42922967Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:15.429315241Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 1.258947644s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.448993352Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:15.449127244Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 738.557325ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.586913645Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 816.758556ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.734074692Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 571.239367ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.842186109Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 1.103751518s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.951754372Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 546.415099ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:15.990723901Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 1.293530589s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.041872355Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.047587703s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.108228164Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 1.053997252s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.136446833Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 990.79138ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.165943845Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 571.157648ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.187571317Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.498267689s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.216849507Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 37.333501ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.246029646Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 45.512604ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.263431284Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:16.26405487Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:16.264302813Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 60.978884ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.298388452Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 1.140858641s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.320336567Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 126.319207ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.333472571Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.469185944s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.348668647Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.267681167s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.368210397Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 255.483164ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.38222869Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 767.336337ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.430215936Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 174.208086ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.473812228Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 789.925924ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.491359758Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 935.957729ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.552593315Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 63.778607ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.56472951Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.012986807s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.578055156Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 91.343368ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.641782089Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 500.73849ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.656797333Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 202.596017ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.671980238Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 877.165304ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.721861989Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 299.997512ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.746315119Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 978.781361ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.772473437Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 1.014601741s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:16.891471866Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 547.48047ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.064176565Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 1.34733082s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.078165298Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.234736159s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.17115519Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 670.138386ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.186098924Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 1.405421037s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.201757014Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.087796086s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.217185472Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 596.762579ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.235656961Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 1.4852283s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.298759357Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 779.158606ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.350377946Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 1.33946721s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.483329018Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 724.833704ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.511606941Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.128459107s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.622414052Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 1.177217866s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.643612849Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 897.051821ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.667841667Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 1.457310909s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.71007461Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.267254854s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.742138998Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 1.346399303s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.768399097Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.22929114s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.836287513Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 723.531133ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.853727771Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 1.450168767s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.874695246Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 1.299051545s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:17.941837334Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 576.399926ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.148853614Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 1.462388573s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.249341933Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.272153097Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"2.45981675s"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 666.558845ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.318016156Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 917.915011ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.420674158Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 840.972427ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.439608982Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:18.440242708Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:18.440505661Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 652.821293ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.584089332Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 1.377586128s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.599273267Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 611.016369ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.622936239Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 733.195866ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.748110641Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.476704184s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.801894562Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 931.956881ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.827967099Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 47.188772ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.842322006Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 826.770439ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.863288731Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 994.107873ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.907257491Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 123.825066ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:18.922456397Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.378127885s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.023907942Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 661.687454ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.06532563Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 208.41022ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.105123098Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 643.063922ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.132173795Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 998.83149ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.153645465Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 954.660276ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.182699772Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 866.505265ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.247084792Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.176410284s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.260334928Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 1.429742991s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.274147969Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 1.443761617s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.291417496Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 545.904948ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.328137042Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 897.084244ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.339074784Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.307643954s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.351687313Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 934.512535ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.500140764Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 948.634448ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.711459518Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 1.137135849s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.7272786Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.055457756s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.743468536Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 806.04501ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.772523064Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 680.337685ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.792313806Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.17194932s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.89427576Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 649.509436ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.918216216Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 1.300707068s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:19.99384902Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 1.407183567s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.131873494Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 1.393530484s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.154141062Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 805.128605ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.168152156Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 921.539819ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.249954753Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.125650062s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.290284587Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.027437774s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.415182145Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 1.418098447s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.445326194Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.043610961s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.454383527Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 818.104284ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.484882329Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 1.421725341s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.581312717Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 1.480323268s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.594437241Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 627.058428ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.673242768Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 903.813406ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.685977019Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 710.255714ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.717197648Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 511.113412ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.824413507Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 872.833974ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.837570401Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 773.406698ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.88034003Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 1.497833535s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:20.987990882Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.238925723s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.080884204Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 706.539734ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.186558756Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 1.274459183s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.308973749Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.031157681s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.321115434Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.07394376s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.331855454Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.280596559s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.346514574Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.023144697s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.430956229Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 1.013270914s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.444431087Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 808.032351ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.455618831Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.377358747s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.51012755Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 568.216725ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.55210067Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 1.369846251s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.562077582Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 1.277891518s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.600807538Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 836.414693ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.665944026Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 884.105795ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.720918299Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 1.173694136s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.87722979Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 1.205345634s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:21.906868843Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 1.396279738s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.015119022Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 1.327345087s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.086361702Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 928.261978ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.133112341Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.411675147s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.276091385Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 981.952023ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.329603563Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.398416421s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.362653231Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 686.204358ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.416151409Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 536.606537ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.466004119Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 814.89308ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.475472196Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 822.486946ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.495632283Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 1.254131965s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.514156353Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 801.905924ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.591897249Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 1.156271368s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.615503301Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 524.612066ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.638477096Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 823.936967ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.891241735Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.355339161s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.901115296Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 976.452959ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:22.943328158Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.06376959s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.016518598Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 1.48006584s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.062689891Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 786.148242ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.074566303Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.411149617s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.11533181Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 1.250522546s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.126851618Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.225582687s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.177325985Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 1.02936123s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.303945052Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.324801375s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.326758746Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 600.335679ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.338283524Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 1.08655399s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.406403551Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 1.266592864s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.423533857Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 756.675059ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.433530559Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.265574955s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.702471243Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 1.204979336s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.727177236Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.324455532s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.779479162Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 908.906563ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.792862059Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 958.792023ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.804911692Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 588.925453ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.874259603Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 760.066782ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.9325342Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 1.000775818s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:23.949839627Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.042803125s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.094238766Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 1.376467428s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.228564892Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.127285591s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.270050256Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 610.98408ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.349184187Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 710.19471ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.391760242Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.173637111s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.414545886Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 1.312561224s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.476804843Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 1.040510325s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.505080293Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 810.865116ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.53895842Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 732.183773ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.555700431Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.324346852s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.66391306Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 516.827752ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.703727557Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 753.802438ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.71468434Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 573.229231ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.72647488Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 1.358402986s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.776161869Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 720.928826ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.9451149Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 945.253083ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.963141754Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 712.655742ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.973003105Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.101409159s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:24.982123719Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 745.871733ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.113263982Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.106050525s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.123750049Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.475264945s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.147050838Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 854.124381ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.230562223Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 1.102269443s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.29859199Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 1.186336032s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.309958477Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 699.757502ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.360780617Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.090339461s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.40896397Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 600.711076ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.478466752Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 534.972134ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.556738953Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.176481721s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.600699254Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 558.967999ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.615729278Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 719.975328ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.677684822Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 863.40711ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.696715167Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 1.285680545s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.801003085Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 708.267822ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.870535847Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 1.388999235s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:25.920167146Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.30003677s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.000059534Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.021616773s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.070470215Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 861.662985ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.089019325Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 785.815392ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.098364481Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 1.356218231s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.107144021Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 1.212315518s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.115064702Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 1.253765165s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.144407163Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 840.314639ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.18414423Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.357798397s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.239062502Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 1.37978529s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.409035983Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 604.947544ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.41761053Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 1.184582143s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.506090917Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.133158222s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.514985438Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 1.211205292s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.569688848Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 705.256415ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.588265489Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.286772023s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.631503151Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.079471004s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.788170986Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.379929496s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.893645546Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 1.365166217s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:26.960211618Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 675.811001ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.031395937Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 567.555737ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.039586901Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 897.540302ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.095355932Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.108579134s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.108038012Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 1.167367235s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.302682366Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.420239723s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.310857039Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 529.708781ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.318493557Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 805.304598ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.336096008Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 620.642605ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.408501949Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 1.043642569s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.473505285Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.380680489s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.606423706Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 539.059716ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.617023015Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 710.971345ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.657846833Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 1.153230358s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.699424879Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 552.225076ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.715237021Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 852.268996ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.788426451Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 706.664997ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.795852627Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 782.288969ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.857789861Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 724.575015ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.876423642Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 533.038658ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:27.928747708Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 1.024023558s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.001921627Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 1.000116444s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.040422822Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 708.56572ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.150822922Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 1.258501167s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.164322531Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.136906554s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.226101713Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 1.490480316s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.277619091Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.282093026s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.285216769Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 1.345277898s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.334386913Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 718.476623ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.377574504Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 684.720834ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.471151403Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 598.429609ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.478852772Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 633.489234ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.511039131Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 764.313209ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.524823663Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 1.031034942s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.609061606Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 1.255977186s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.647784522Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 831.859633ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.656305929Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 729.684289ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.740031267Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 1.13329589s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.861068277Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a\") failed: Property not found. retry in 1.222462112s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.869014988Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 877.391679ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:28.99615707Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 1.176124843s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.03520427Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 648.236903ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.049188233Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 680.752248ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.086294683Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 1.295774784s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.104367639Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 565.788616ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.153268969Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 863.897073ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.163813537Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 967.72028ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.298931041Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87\") failed: Property not found. retry in 1.145631407s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.371375623Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b\") failed: Property not found. retry in 641.003484ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.432727442Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 947.995974ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.479617862Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 1.059166348s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.497876749Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c\") failed: Property not found. retry in 1.474053408s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.576980109Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c\") failed: Property not found. retry in 745.8462ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.647879205Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 952.070931ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.688472711Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586\") failed: Property not found. retry in 1.462480157s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.706017381Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 1.319079946s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.726911415Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 1.004951238s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.774791805Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738\") failed: Property not found. retry in 1.011763786s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.782255281Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7\") failed: Property not found. retry in 1.460482158s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.883473578Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4\") failed: Property not found. retry in 1.352561976s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:29.913437125Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb\") failed: Property not found. retry in 759.284546ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.004865381Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386\") failed: Property not found. retry in 1.05974926s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.037041191Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7\") failed: Property not found. retry in 618.221612ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.224855495Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8\") failed: Property not found. retry in 868.232574ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.277952618Z","hostname":"sock","pid":1572,"zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\") failed: Property not found. retry in 648.164554ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.4187528Z","hostname":"sock","pid":1572,"zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 540.376715ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.437429831Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86\") failed: Property not found. retry in 940.12013ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.567070479Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:30.627296716Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49845"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:30.629271456Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"84fd9ce4-ad6e-4898-9076-5bf842869a62","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49845","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":659,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988\") failed: Property not found. retry in 849.66252ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.650991319Z","hostname":"sock","pid":1572,"zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 990.090577ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:30.7891838Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3\") failed: Property not found. retry in 1.328759942s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:31.221185559Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 560.444296ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:31.402098761Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c\") failed: Property not found. retry in 776.472748ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:31.791674811Z","hostname":"sock","pid":1572,"zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 703.110377ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:31.972692496Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 518.113745ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:32.694133954Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.436111904s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:33.231535838Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.241687799s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:34.692481551Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:35.812703954Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"3.142985321s"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.070552566s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:35.949373324Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 843.518088ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:37.029736369Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.424848507s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:37.883715696Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:38.990138577Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"5.305553412s"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 786.306849ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:39.320910365Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 858.830246ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:40.116193301Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 558.35586ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:40.984885918Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 987.557765ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:41.554141098Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 865.256396ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:42.55214771Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 810.906019ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:43.427309593Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.152796026s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:44.249990329Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:44.331957259Z","hostname":"sock","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"18.210721394s"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 1.045169045s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:45.413140052Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f\") failed: Property not found. retry in 776.069626ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:14:46.470497921Z","hostname":"sock","pid":1572,"zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285243156Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_2c60990c-614d-44ea-a7bf-2bae04edc1b3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285313887Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_dde78775-b16f-45ca-91b2-9608c686b32c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285320647Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_d634556d-8226-4df6-8dfc-09c66049e16b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285326156Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_external_dns_7ed51b56-e87d-4a9a-b9bc-27ece36a86d7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285331467Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_1ee7d195-9a59-4752-ad62-863aa8d03aa4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285335447Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_065751ee-697c-4a72-8cf7-687dac3ede9c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285339337Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_8377dba6-b489-43b3-98e1-05bb7df98988"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285343487Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_a4913a46-2b97-4d0c-965d-3da766243586"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285347307Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_32137949-0a38-4d1e-a737-2d07177c3738"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285351227Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_external_dns_b2beb08a-ae3f-469f-86c4-36645336e7c7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285355337Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_1e02403f-00b9-457b-a823-4da05e840386"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285359217Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_511547b5-244c-4c59-b939-8d95795a2c87"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285391257Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_e2b91b21-636b-4131-93b7-4d20d375703a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285395417Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_oximeter_4ce79729-70af-4ea9-8907-92eccad979f8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285399307Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_1e4bfc38-893c-408c-9d22-948390ba9d86"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285403217Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_1f4e8f94-7bc7-4b29-b92c-ee717847f51c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285406937Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_clickhouse_12eca9ce-450a-47c6-aa79-35164c1f047f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285410428Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_7f59615c-c60c-43bf-8c6b-f46c94caeddb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.285414137Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_c559bdc6-d957-40c2-ad54-ae7947822ea7"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.286294057Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.290122156Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.292143636Z","hostname":"sock","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1d8fad99-c34b-416c-aa53-c17fe306dd9b","remote_addr":"[fd00:1122:3344:101::1]:44733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":45029165,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.299896446Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1322"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.300223649Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.300537822Z","hostname":"sock","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.308742216Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:708"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.310276152Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:720"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:47.347588074Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bf6bc666-c3c5-4b42-bff6-051525a8f889\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:48 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"bf6bc666-c3c5-4b42-bff6-051525a8f889\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:48.475000641Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1514bc0-b5c6-468a-8e33-12fec3c672ff\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:49 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"e1514bc0-b5c6-468a-8e33-12fec3c672ff\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:49.178395665Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4b070543-185e-4c5a-93cc-b9910e6c4430\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:49 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"4b070543-185e-4c5a-93cc-b9910e6c4430\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:49.602754902Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"470d967a-09e2-4d3e-a3cd-b69922caca2d\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:50 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"470d967a-09e2-4d3e-a3cd-b69922caca2d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:50.208055191Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7bef6f07-0f85-4228-ad68-0e541cfda12a\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:50 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"7bef6f07-0f85-4228-ad68-0e541cfda12a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:50.887286358Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"99de20f9-5c3a-4798-aa66-17091a6089ab\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:51 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"99de20f9-5c3a-4798-aa66-17091a6089ab\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:51.303869514Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c8f86926-3263-4faa-a216-ff5e9d1c6710\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:51 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"c8f86926-3263-4faa-a216-ff5e9d1c6710\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:51.827104303Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d62a61f3-7640-4511-8813-f5668d4f72c4\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:51 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"d62a61f3-7640-4511-8813-f5668d4f72c4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:52.414337718Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88269fac-003c-4ff7-93d6-d585b92beb37\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:52 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"88269fac-003c-4ff7-93d6-d585b92beb37\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:53.049082648Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d9f5d99e-d559-4852-b9b6-f24aff0f26b5\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:53 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"d9f5d99e-d559-4852-b9b6-f24aff0f26b5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:53.637593176Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b704df44-738f-49e3-ba47-a3d4195a8f1c\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:54 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"b704df44-738f-49e3-ba47-a3d4195a8f1c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:54.623482003Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20985e1c-2dce-4051-8356-69e5daeebae7\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:55 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"20985e1c-2dce-4051-8356-69e5daeebae7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:55.551472857Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c819ea48-aa7e-4d5e-8c31-7a7ebf55a304\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:55 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"c819ea48-aa7e-4d5e-8c31-7a7ebf55a304\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:56.439893547Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cc3db658-e74a-490c-b719-bbf1ccc273e0\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:57 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"cc3db658-e74a-490c-b719-bbf1ccc273e0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:57.31117254Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"974919b2-b629-4c01-8c0f-b0b1b61453e3\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:57 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"974919b2-b629-4c01-8c0f-b0b1b61453e3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:58.148961611Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a5d079b1-ae1d-4f77-baa8-0e5a1d647217\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:14:59 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"a5d079b1-ae1d-4f77-baa8-0e5a1d647217\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:14:59.166710185Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:00.673161704Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60820"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:00.673251684Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"82940038-c41d-45b7-9547-9e9458a7f880","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60820","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0b25084-3d14-4a4c-aa61-4aa4cbfd67d3\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:15:00 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"f0b25084-3d14-4a4c-aa61-4aa4cbfd67d3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:01.0027949Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"29555811-8b17-4459-97bd-cf2014c4e5fd\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:15:03 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"29555811-8b17-4459-97bd-cf2014c4e5fd\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:03.181581024Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"972b28bc-5e7d-4f60-ac6d-0860660d783a\", \"content-length\": \"173\", \"date\": \"Thu, 31 Oct 2024 00:15:04 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7\\\"\", request_id: \"972b28bc-5e7d-4f60-ac6d-0860660d783a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:04.854513098Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:05.579364232Z","hostname":"sock","pid":1572,"component":"NexusNotifierTask","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:05.678503478Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39169"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:05.680280056Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:05.680289116Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:05.681289736Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:05.682724061Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:05.867364102Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"a5ea8669-f19c-4a2c-816a-c5d4ddde47c2","remote_addr":"[fd00:1122:3344:101::a]:39169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":188523,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:08.082591291Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:40997","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::d]:50335"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:08.109310784Z","hostname":"sock","pid":1572,"uri":"/f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","method":"GET","req_id":"a57162a2-f34c-4141-ab78-0e09b2cb70d8","remote_addr":"[fd00:1122:3344:101::d]:50335","local_addr":"[fd00:1122:3344:101::1]:40997","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":25234,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.183534586Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47812"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.189290165Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name(\"external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name(\"external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name(\"nexus-1e4bfc38-893c-408c-9d22-948390ba9d86\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name(\"nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name(\"nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.189344426Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.189529228Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.189533857Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.189562758Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.189570478Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.189582388Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.189609028Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ac5ccf05-c15c-4abb-bf89-89961eeb0e47","remote_addr":"[fd00:1122:3344:101::a]:47812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5830,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.191275615Z","hostname":"sock","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.44953999Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.449735692Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.449745452Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.449803823Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.450131546Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:09.641236564Z","hostname":"sock","pid":1572,"uri":"/inventory","method":"GET","req_id":"181665ad-e73d-456a-8aad-fc995a31a6cf","remote_addr":"[fd00:1122:3344:101::a]:40258","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":191548,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.786795527Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56715"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.78708435Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name(\"external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name(\"external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name(\"nexus-1e4bfc38-893c-408c-9d22-948390ba9d86\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name(\"nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name(\"nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.78712938Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.78715191Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.78716579Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.787177391Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.787199451Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.787220301Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.787248181Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9f941524-701e-4e52-be50-a6bac6d1decd","remote_addr":"[fd00:1122:3344:101::b]:56715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":330,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795359424Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:57926"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795542086Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name(\"external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name(\"external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name(\"nexus-1e4bfc38-893c-408c-9d22-948390ba9d86\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name(\"nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name(\"nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795586657Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795615247Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795624047Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795641287Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795660217Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795677707Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.795694098Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2b53a1c8-d913-4ac7-9af6-34f11659a1c1","remote_addr":"[fd00:1122:3344:101::a]:57926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":246,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.847926153Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40636"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.848102915Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1ea2ccb0-4d8d-450a-b5fa-20f11d99c6b4, kind: Service { id: 7ed51b56-e87d-4a9a-b9bc-27ece36a86d7 }, name: Name(\"external-dns-7ed51b56-e87d-4a9a-b9bc-27ece36a86d7\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 486dd504-bd2f-498e-8990-b1d7d833ab10, kind: Service { id: b2beb08a-ae3f-469f-86c4-36645336e7c7 }, name: Name(\"external-dns-b2beb08a-ae3f-469f-86c4-36645336e7c7\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 100])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1e6d66a8-381f-4d1d-a719-7f1f1a2fab2f, kind: Service { id: 1e4bfc38-893c-408c-9d22-948390ba9d86 }, name: Name(\"nexus-1e4bfc38-893c-408c-9d22-948390ba9d86\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 239, 133])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 397cb5ae-79f3-4840-95d1-a308d95a2fad, kind: Service { id: 2c60990c-614d-44ea-a7bf-2bae04edc1b3 }, name: Name(\"nexus-2c60990c-614d-44ea-a7bf-2bae04edc1b3\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 222])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f546a11c-0e59-4819-8b6f-c01e0cfa393a, kind: Service { id: 1f4e8f94-7bc7-4b29-b92c-ee717847f51c }, name: Name(\"nexus-1f4e8f94-7bc7-4b29-b92c-ee717847f51c\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 148])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.848129595Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.848140005Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.848150435Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.848153955Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.848161495Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.848169945Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:10.848182385Z","hostname":"sock","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"58e3ff63-0302-4f2b-98e8-35e85f3e0764","remote_addr":"[fd00:1122:3344:101::c]:40636","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:15.699850855Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42260"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:15.700470392Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"4bc204aa-70cf-4aa2-8088-332f4b7f08dc","remote_addr":"[fd00:1122:3344:101::a]:42260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":535,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:15.722499257Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:15.722515867Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"84df363d-5565-46b0-afcc-1a6f742edaae","remote_addr":"[fd00:1122:3344:101::a]:42260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1153,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:15.813331138Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"50cefdd5-7162-476a-b97d-f763d9f99f8d","remote_addr":"[fd00:1122:3344:101::a]:42260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3108,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.281963697Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33315"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.282079209Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fb5a81c5-fac3-4e0e-8b45-332d4661a20e","remote_addr":"[fd00:1122:3344:101::c]:33315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.39742278Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:34986"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.397666622Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"fef3ae48-d56c-4521-9699-e550b91f132e","remote_addr":"[fd00:1122:3344:101::c]:34986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":184,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.447480252Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.447504873Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b3622922-c60a-4698-9f26-b4cb988cc064","remote_addr":"[fd00:1122:3344:101::c]:33315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.505603598Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b3a07f14-25f5-4c5e-ae39-965dff06168b","remote_addr":"[fd00:1122:3344:101::c]:33315","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":172,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.627473006Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:34371"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.628002652Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.698829487Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.767577581Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c8207f66-4015-4a65-8d2b-4baefbdafa29","remote_addr":"[fd00:1122:3344:101::c]:34371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140002,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.769346829Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"725d98e0-d9e3-4a68-a0fd-0b1249f44136","remote_addr":"[fd00:1122:3344:101::c]:34371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1470,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.804493659Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.804623861Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.805904303Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.807158526Z","hostname":"sock","pid":1572,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:16.807174187Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"5d276de4-b1e2-4917-b5f6-ce71842f99b5","remote_addr":"[fd00:1122:3344:101::c]:34371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3829,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.063768755Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41619"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.063888896Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6a461945-b2fc-43fc-9359-3f41f6490737","remote_addr":"[fd00:1122:3344:101::b]:41619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.096208767Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.096239517Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"61a02718-a8bb-41f5-864a-4a7ee44dd55f","remote_addr":"[fd00:1122:3344:101::b]:41619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.134737702Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.134878163Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"604a4501-7d70-40be-b44c-e5ca51949eb0","remote_addr":"[fd00:1122:3344:101::a]:59344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.219088045Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.219121166Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a43dae92-4620-4720-97a8-30756f0e7b03","remote_addr":"[fd00:1122:3344:101::a]:59344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.352995667Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4c98b93a-1bec-4660-9ec1-7334884258b0","remote_addr":"[fd00:1122:3344:101::b]:41619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":174,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.394657724Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7350159b-758f-478a-bc6e-05443bd79d44","remote_addr":"[fd00:1122:3344:101::a]:59344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":173,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.424505969Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.424692131Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3f66707b-d5b2-4766-a070-afcaa52de888","remote_addr":"[fd00:1122:3344:101::a]:42294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.429636972Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36286"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.429736623Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.474652143Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.541558008Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7d3b056b-e450-407e-a639-bcba9d3d8239","remote_addr":"[fd00:1122:3344:101::a]:36286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111847,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.54172631Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60352"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.54177015Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.542097234Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a0d50dd2-ae59-44eb-811c-f28745e1fdfd","remote_addr":"[fd00:1122:3344:101::a]:36286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":239,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.542108014Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.547321457Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"bf590cdd-8acb-423c-bc2c-387f99713ea8","remote_addr":"[fd00:1122:3344:101::b]:49324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5383,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.587958693Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.653313063Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"aefa3b7f-688e-4adb-9a70-68db3b815dc0","remote_addr":"[fd00:1122:3344:101::b]:60352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111320,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:17.653777998Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"26748ca7-f6a7-4272-8edc-fc6d4e6ddc0e","remote_addr":"[fd00:1122:3344:101::b]:60352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:30.708512753Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47127"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:30.708853596Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"4403c0d4-a303-4436-a0d6-c2e033a88ae9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47127","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.229247695Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56966"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.229417297Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ccc8c684-e9e4-4525-938b-620ddb69a0b5","remote_addr":"[fd00:1122:3344:101::c]:56966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.291936887Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.291968177Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4a941008-3932-476e-bd27-074280b764d6","remote_addr":"[fd00:1122:3344:101::c]:56966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.301639967Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9d63f4bc-b15e-466a-ac74-4c06b4d8adbf","remote_addr":"[fd00:1122:3344:101::c]:56966","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.408382709Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.408520211Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"482abf45-2255-48fd-a35b-ee820fa3c126","remote_addr":"[fd00:1122:3344:101::c]:64119","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.423527005Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60713"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.423633136Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.464654976Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.521596259Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"574c7b10-12b4-47e2-a94b-478f73d8c0ec","remote_addr":"[fd00:1122:3344:101::c]:60713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97996,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:46.521886332Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a806bec3-2160-499b-a324-7dd116b991a4","remote_addr":"[fd00:1122:3344:101::c]:60713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.08800637Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.088399584Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5be28f52-32f7-42c5-85fc-eaf32ee25806","remote_addr":"[fd00:1122:3344:101::b]:59381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.143950743Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.143978643Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cb1dd103-a044-4bf7-959b-0cdb1e8ca276","remote_addr":"[fd00:1122:3344:101::b]:59381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.155220409Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f05a1d75-ab22-4102-b9fe-f0278149fa20","remote_addr":"[fd00:1122:3344:101::b]:59381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":271,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.255490236Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41003"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.255674907Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"26c2e6cc-9a30-40f3-92fd-bb7d73f98829","remote_addr":"[fd00:1122:3344:101::b]:41003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.279048207Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:37141"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.279153018Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.316892805Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.39262969Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"281eb10e-79fd-429f-a541-9cd59fd04ee6","remote_addr":"[fd00:1122:3344:101::b]:37141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113519,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.392923943Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:50752"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.39852984Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"382ac351-9b43-43f5-801a-fe5eec4b0f76","remote_addr":"[fd00:1122:3344:101::b]:37141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5474,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.398639302Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"41aaaa1e-60d4-45c1-bd4c-1ae878394403","remote_addr":"[fd00:1122:3344:101::a]:50752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":571,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.447380251Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.447412271Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"adcc8c90-bc37-413d-a7d7-396fc3e461d7","remote_addr":"[fd00:1122:3344:101::a]:50752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.457752617Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3a77c446-b55a-4da2-96c7-74c4f9b5630c","remote_addr":"[fd00:1122:3344:101::a]:50752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":264,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.472568729Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59884"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.472661749Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.508474226Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.565564801Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9e16bb8a-c2a0-44d5-a765-ad0081004eef","remote_addr":"[fd00:1122:3344:101::a]:59884","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92938,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.565682552Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.565929325Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0cea5cae-d213-411a-8da1-8747ddb3c2fb","remote_addr":"[fd00:1122:3344:101::a]:44936","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":166,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:15:47.565962295Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"dc0c0e9a-5c11-477e-ada3-bfc777699b4b","remote_addr":"[fd00:1122:3344:101::a]:59884","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:00.741738841Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56363"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:00.741962043Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"0ff2c40f-47db-4916-9d95-464805a56418","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56363","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.159034753Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:53173"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.159179215Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c4c885d2-3328-467e-a5f4-dec5f9574e1c","remote_addr":"[fd00:1122:3344:101::c]:53173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.204985694Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.205047055Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8e920e5f-c0d5-4c7e-b15a-45ce676ef62a","remote_addr":"[fd00:1122:3344:101::c]:53173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.213926745Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4854324e-7365-4825-81b6-e78044319fd7","remote_addr":"[fd00:1122:3344:101::c]:53173","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.3696054Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54059"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.369765471Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"51c5205d-b5ed-468b-8da2-8406515522e4","remote_addr":"[fd00:1122:3344:101::c]:54059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.416864584Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37056"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.416947345Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.453748322Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.513751206Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ba72233e-3edd-4549-8317-b01d185b538a","remote_addr":"[fd00:1122:3344:101::c]:37056","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96839,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.51410304Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f006bf94-5ed0-4d6a-b956-3d868e659180","remote_addr":"[fd00:1122:3344:101::c]:37056","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.924011998Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55802"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.92416822Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e1b27756-7745-4b30-8d89-bfe776da047d","remote_addr":"[fd00:1122:3344:101::b]:55802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.978806699Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.97884295Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"91b796c6-f31d-456b-baf8-1d514d348862","remote_addr":"[fd00:1122:3344:101::b]:55802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:16.989288477Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"40f98c0f-3605-4ad2-9b40-bcf7dd6f2494","remote_addr":"[fd00:1122:3344:101::b]:55802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":244,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.204081937Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.204341629Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"2c27c882-8d8c-42f6-8603-417ab04b864b","remote_addr":"[fd00:1122:3344:101::b]:46862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":203,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.212155379Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62701"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.212271151Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8afda944-e752-4f14-8b73-66f35c54829c","remote_addr":"[fd00:1122:3344:101::a]:62701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.227248564Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62177"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.227366655Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.269512137Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.336619994Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"1767e4c3-098e-423d-8819-0a4bcb2b0c04","remote_addr":"[fd00:1122:3344:101::b]:62177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109283,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.336877867Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49193"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.336914177Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.336964188Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"65ea1c6e-084b-4658-9a52-78e1ff22038f","remote_addr":"[fd00:1122:3344:101::a]:62701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.33716137Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4a0d758f-365f-4d48-b39f-4f5c65ccfaaa","remote_addr":"[fd00:1122:3344:101::b]:62177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":278,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.33719025Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.34789635Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"38902c43-3044-4c06-8e60-3b5864bb0292","remote_addr":"[fd00:1122:3344:101::a]:62701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":351,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.377855946Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.409819643Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55592"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.410088826Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"43d52f2c-6efd-4437-81f6-cfaabf19a9da","remote_addr":"[fd00:1122:3344:101::a]:55592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.437026442Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"56bf7302-d008-4263-9002-782f9713b11c","remote_addr":"[fd00:1122:3344:101::a]:49193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99951,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:17.437346425Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"485b52c5-3a53-492a-8d10-9e3ba6b9aad1","remote_addr":"[fd00:1122:3344:101::a]:49193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:30.778516845Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:30.778616746Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"5fecfe4f-ff97-4dc3-b99f-883bd4057693","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41939","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.358038719Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.35817056Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ff869294-e24d-4da3-983b-adf272890e9d","remote_addr":"[fd00:1122:3344:101::c]:56607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.405783378Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.405800108Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"668bb66d-4d53-43af-96b2-127e62cf17c7","remote_addr":"[fd00:1122:3344:101::c]:56607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.418000393Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:35041"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.418174575Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"c35af1dd-2db7-45f5-85fa-08bef3a76c5e","remote_addr":"[fd00:1122:3344:101::c]:35041","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.424753792Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c6ccde08-eecf-45e2-88a6-75dc40679f87","remote_addr":"[fd00:1122:3344:101::c]:56607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.500232085Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56109"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.500337036Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.540199405Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.598709394Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"04ca7f76-1341-40f5-9b20-3fb115a5a5e5","remote_addr":"[fd00:1122:3344:101::c]:56109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98418,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:46.599045127Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"08c2a65c-b410-46ac-bf73-fbcb2001fbd8","remote_addr":"[fd00:1122:3344:101::c]:56109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.052078867Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44093"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.052270989Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ca75707f-ead7-4ce5-97aa-71369ddbe92b","remote_addr":"[fd00:1122:3344:101::b]:44093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.155019882Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.155057352Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"063c39e5-29b5-4c8f-900f-0006471b346e","remote_addr":"[fd00:1122:3344:101::b]:44093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.169590721Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"64204f58-fb39-4d5f-b732-d6f8865b98f2","remote_addr":"[fd00:1122:3344:101::b]:44093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":222,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.317996871Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.318173313Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"e1d57f0f-8530-46aa-a55a-a27a51d8057d","remote_addr":"[fd00:1122:3344:101::b]:52084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.342878056Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42679"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.342957536Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.380698003Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.437914879Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b3aeae6b-d9cb-42ad-b90a-34cd7ccd456a","remote_addr":"[fd00:1122:3344:101::b]:42679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94992,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.438158941Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36371"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.438239092Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55167"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.438444614Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"92ea80c9-3ebd-4b50-9c51-45b6b028258c","remote_addr":"[fd00:1122:3344:101::b]:42679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":218,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.438551705Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"39dac58a-703f-4288-afaa-587a5cd779da","remote_addr":"[fd00:1122:3344:101::a]:36371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":151,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.438649936Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"b5636f17-f996-4dd0-b4fe-a505f79c496d","remote_addr":"[fd00:1122:3344:101::a]:55167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":238,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.48881547Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.48884508Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b34f8753-2f91-4039-a8bf-ad1d73c79b5d","remote_addr":"[fd00:1122:3344:101::a]:36371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.498680461Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4d8ccea1-c778-4f58-888b-a279c75a916c","remote_addr":"[fd00:1122:3344:101::a]:36371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":267,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.623429979Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54259"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.62354262Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.659934782Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.710699922Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e173e7f7-d033-4e80-a813-bbfc77517c8f","remote_addr":"[fd00:1122:3344:101::a]:54259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87201,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:16:47.711162087Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"ddc2bb86-806b-4000-84b4-c3368720124a","remote_addr":"[fd00:1122:3344:101::a]:54259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:00.810600421Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45644"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:00.810698652Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"89e1b494-0426-4470-97f9-5d7e3a2bf21b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45644","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.209208242Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36156"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.209347903Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2b185e99-ce5f-4e9a-8723-043116643412","remote_addr":"[fd00:1122:3344:101::c]:36156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.251217572Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.251244072Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2e453a5d-13cb-4f6f-ada8-b2e36befa97f","remote_addr":"[fd00:1122:3344:101::c]:36156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.264470698Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f943b0c0-a087-475a-a04e-ea10313bbe03","remote_addr":"[fd00:1122:3344:101::c]:36156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":203,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.305322526Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47558"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.305545709Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"0962c029-2293-4d6b-99db-c49eb9c9393c","remote_addr":"[fd00:1122:3344:101::c]:47558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":149,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.359599692Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62081"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.359714214Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.397004385Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.449558323Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"915ec286-c4b5-43e3-8c12-f095a8db0858","remote_addr":"[fd00:1122:3344:101::c]:62081","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89899,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:16.449846026Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e7d693ca-f5e0-481c-b271-accf0a74304b","remote_addr":"[fd00:1122:3344:101::c]:62081","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.048335416Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48385"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.048516398Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"36be1ee2-0bab-47b8-bb46-e08a803cc9d9","remote_addr":"[fd00:1122:3344:101::b]:48385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.092897133Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.092940553Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"26067b8f-d7da-44a5-b71f-e5573e8bf504","remote_addr":"[fd00:1122:3344:101::b]:48385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.10340984Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c2fda985-f297-4a96-b4eb-8315a2125083","remote_addr":"[fd00:1122:3344:101::b]:48385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":323,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.154809297Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.154943448Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"27188a28-9d78-4a55-91f6-e93bdd1403e8","remote_addr":"[fd00:1122:3344:101::a]:65198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.164108692Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51357"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.164321654Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"4a14f129-5596-48b7-a320-60f9601f3ad6","remote_addr":"[fd00:1122:3344:101::b]:51357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":136,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.173100174Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.173118774Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6537f32e-22db-4d71-87d0-42c43c945047","remote_addr":"[fd00:1122:3344:101::a]:65198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.181748223Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"19789701-1a86-4d56-aa26-01ec1e2eddfc","remote_addr":"[fd00:1122:3344:101::a]:65198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.187060327Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62444"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.187155658Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.23024814Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.293349306Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"bffded0e-0392-49c0-befc-544f2f4adec2","remote_addr":"[fd00:1122:3344:101::b]:62444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106234,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.293665229Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"40b7898d-5d8c-4c30-a8e0-724d2896aefb","remote_addr":"[fd00:1122:3344:101::b]:62444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.308550662Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51415"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.308742214Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.34644583Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.404951629Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5465e6b0-b3de-4986-90d1-d37b7e38dd6f","remote_addr":"[fd00:1122:3344:101::a]:51415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96280,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.405185391Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.405471444Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"6c7994c7-ea83-4437-bcb7-c6232a7d0922","remote_addr":"[fd00:1122:3344:101::a]:51415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":234,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:17.405494574Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"3251646a-6a25-4f5c-8a0c-83eea247690c","remote_addr":"[fd00:1122:3344:101::a]:55034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":190,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:30.843227975Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61223"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:30.843333336Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"e3108823-5ff3-4561-8ca5-73c81e9f3d2f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61223","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:39.719576356Z","hostname":"sock","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.395856044Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47207"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.396061306Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9c6e3d99-b836-433d-a11b-d95c8bb25ebf","remote_addr":"[fd00:1122:3344:101::c]:47207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.445553163Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.445585163Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d9bb2dcb-345c-4fb1-a9b0-3414f33a1e9e","remote_addr":"[fd00:1122:3344:101::c]:47207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.455834048Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a34775b8-ad76-41d5-8e9d-74c3ef498574","remote_addr":"[fd00:1122:3344:101::c]:47207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":197,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.535671746Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.535849927Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"bac86bd6-e848-4b08-bf19-975b35051db2","remote_addr":"[fd00:1122:3344:101::c]:54465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.567469091Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52986"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.567572522Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.601749462Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.656980018Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"bf7f5237-9618-42dc-bb58-17b13fd463c7","remote_addr":"[fd00:1122:3344:101::c]:52986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89451,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:46.657329102Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"859282ba-2484-464c-b0c1-f9574b982c93","remote_addr":"[fd00:1122:3344:101::c]:52986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.163994991Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.164153963Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2907c529-b93c-4629-b2bc-d7fea9d0a620","remote_addr":"[fd00:1122:3344:101::b]:58215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.220244067Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.220272397Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"181c9357-f851-4bc8-92eb-af6d3274e1e8","remote_addr":"[fd00:1122:3344:101::b]:58215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.239340593Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.239550375Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"920fcf06-ab59-4a11-8730-d555c63e8c02","remote_addr":"[fd00:1122:3344:101::b]:58300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":164,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.249189004Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ed1efd67-cca8-479f-bc54-601dc9c7a3ea","remote_addr":"[fd00:1122:3344:101::b]:58215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":176,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.298196995Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42144"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.298282876Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.33571576Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.39039223Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"bc7de81c-7896-4ab7-b970-bcdf3a1447cd","remote_addr":"[fd00:1122:3344:101::b]:42144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92147,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.390708243Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"931a01bc-75f5-44b6-82c4-193e479e1de4","remote_addr":"[fd00:1122:3344:101::b]:42144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.497856291Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60714"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.497929291Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b5f539a6-55fc-47c4-890c-da17af788782","remote_addr":"[fd00:1122:3344:101::a]:60714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.517423321Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.517440971Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f0d4c23a-765f-45cf-8b11-02e9ccb819fe","remote_addr":"[fd00:1122:3344:101::a]:60714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.526316472Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"29bdf832-f4f7-4935-8bcf-4d2c5ab7451e","remote_addr":"[fd00:1122:3344:101::a]:60714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.613332503Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.613519345Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"c7ee502a-7c61-4a5b-9f4d-8f49126a3ae9","remote_addr":"[fd00:1122:3344:101::a]:56389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.615655327Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65297"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.615744598Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.658335094Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.718728703Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"59db88dd-b082-493e-8e63-002ec500a158","remote_addr":"[fd00:1122:3344:101::a]:65297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103012,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:17:47.719015116Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4111934a-879c-4a64-9de9-98a326f5abae","remote_addr":"[fd00:1122:3344:101::a]:65297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:00.878485424Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46578"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:00.878601725Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"0dc9be49-b60d-44ad-8dd9-add422e8e9e2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46578","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.183887821Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63559"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.184060513Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2f185e7c-9b12-462c-b747-1845f28faf07","remote_addr":"[fd00:1122:3344:101::c]:63559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.225536237Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.225579808Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d1871b01-8084-4ac4-8b41-5f70a0299ef7","remote_addr":"[fd00:1122:3344:101::c]:63559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.233234536Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"55ba7333-16f5-4bb8-8ef1-ce34ec9b3397","remote_addr":"[fd00:1122:3344:101::c]:63559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.293874587Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45621"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.293966258Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.332835726Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.386681878Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7acb0d66-58a3-4999-bac4-e9d47cb0a26a","remote_addr":"[fd00:1122:3344:101::c]:45621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92749,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.386863619Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64651"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.38695862Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"9e7b278e-e23e-47d5-a008-fe371905d6e5","remote_addr":"[fd00:1122:3344:101::c]:45621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:16.387100622Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"27faff21-9101-4880-9e51-69ffacd884d9","remote_addr":"[fd00:1122:3344:101::c]:64651","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.041411654Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:63150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.041566805Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0f46f3e4-64cb-480e-a744-2afa0cdd4f1b","remote_addr":"[fd00:1122:3344:101::b]:63150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.064683832Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:63825"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.064926424Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"aca78501-7e48-44f5-9a6b-b99ce8bc2183","remote_addr":"[fd00:1122:3344:101::b]:63825","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":183,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.092345785Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.092388016Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a8a2bce8-ed24-45d1-9782-fcf7d958019b","remote_addr":"[fd00:1122:3344:101::b]:63150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.101282787Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b3bb0dd2-38cd-46eb-ab77-1c9fda3f0af4","remote_addr":"[fd00:1122:3344:101::b]:63150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.15631896Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59674"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.156438941Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.196138258Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.255114362Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fc165c57-6eb2-42cd-abac-1886a922e2d1","remote_addr":"[fd00:1122:3344:101::b]:59674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98729,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.255393315Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"9b53f7f9-df3b-455b-a4d7-e40f3cc081a4","remote_addr":"[fd00:1122:3344:101::b]:59674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.303903692Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.304037983Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"57143d5d-9cff-4882-83a2-ece99d79ef30","remote_addr":"[fd00:1122:3344:101::a]:61251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":67,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.329210741Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.329232751Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1b3fc3d8-eab4-4a10-b179-feed38f800d8","remote_addr":"[fd00:1122:3344:101::a]:61251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.338343935Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"75f6dd86-87fb-44a7-888b-c80136d2efaf","remote_addr":"[fd00:1122:3344:101::a]:61251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.363427441Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55033"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.363507822Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.40327316Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.459615546Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"15a847de-f8bd-4233-99d9-436fef4e9858","remote_addr":"[fd00:1122:3344:101::a]:55033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96134,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.459808568Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:64593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.459884649Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"47c6cb76-7bee-4fe5-ae51-38295fa1395d","remote_addr":"[fd00:1122:3344:101::a]:55033","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:17.460038911Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"a2ad7f8d-77b8-4a9b-87c8-6098c5c3832d","remote_addr":"[fd00:1122:3344:101::a]:64593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":136,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:30.911442949Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60844"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:30.91158807Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"4035a558-59a9-4c28-8b0d-63995b96e56a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60844","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.384590378Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.3848227Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"04406ea5-b447-4ce1-8f17-10682b00a472","remote_addr":"[fd00:1122:3344:101::a]:55035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.386950342Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(15018464) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.386982243Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"PUT","req_id":"beb61adc-ea44-4bfc-a3c9-0db6125db158","remote_addr":"[fd00:1122:3344:101::a]:55035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.565677583Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40776"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.572577394Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 581b9a38-aa7e-48a2-bd72-3e109217b461, project_id: b8fce7e8-d453-462f-adc8-3733e1c4452d }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-31T00:18:38.565403590Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15018464))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-f15630e35db\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: cb360434-786a-4a5f-b2cb-17b2f9af1a5f, read_only_parent: Some(Volume { block_size: 512, id: f4843286-27a6-4342-aff1-c0d8c16c1dd7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 246c3072-1690-4f6f-adf8-5c44a3742362, key: Some(\"7xNaLJhQXfLIWc34eKqxqf5Z6MSLemeqY4kg6Q8IOl8=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::12]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: cb360434-786a-4a5f-b2cb-17b2f9af1a5f, key: Some(\"MFxAp2a/h6Pto4lsEvm8yEWCoSW44SJldq3tI6+4v0k=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19000\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-f15630e35db\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.572638084Z","hostname":"sock","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.572877386Z","hostname":"sock","pid":1572,"propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1281","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15018464))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 487b7899-43a4-486a-bd8d-ca0047fd7478, kind: Instance { id: 054c3697-a46c-4a3c-95ef-4a8d9a5537c2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15018464), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-f15630e35db\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: cb360434-786a-4a5f-b2cb-17b2f9af1a5f, read_only_parent: Some(Volume { block_size: 512, id: f4843286-27a6-4342-aff1-c0d8c16c1dd7, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 246c3072-1690-4f6f-adf8-5c44a3742362, key: Some(\"7xNaLJhQXfLIWc34eKqxqf5Z6MSLemeqY4kg6Q8IOl8=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::12]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: cb360434-786a-4a5f-b2cb-17b2f9af1a5f, key: Some(\"MFxAp2a/h6Pto4lsEvm8yEWCoSW44SJldq3tI6+4v0k=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19000\", \"[fd00:1122:3344:101::15]:19000\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-f15630e35db\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-31T00:18:38.565403590Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.574450913Z","hostname":"sock","pid":1572,"uri":"/vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","method":"PUT","req_id":"ebd08d81-2c4d-4fb8-91dc-0f8291ef6f01","remote_addr":"[fd00:1122:3344:101::a]:40776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8700,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.686969645Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.687221948Z","hostname":"sock","pid":1572,"uri":"/vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state","method":"PUT","req_id":"3031d5d6-ac8e-4ec0-9d5f-ee99090e4996","remote_addr":"[fd00:1122:3344:101::a]:43117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":147,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.691101768Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 15018464 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.691249559Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.729619762Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.751242853Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.763679011Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:57528"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.763872423Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1ea72172-3257-4295-b006-f3d2f79f2a00","remote_addr":"[fd00:1122:3344:101::a]:57528","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:38.789227233Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {487b7899-43a4-486a-bd8d-ca0047fd7478: {10.151.2.175: {1865c5a2-f061-4256-aaff-e92277867544}, 10.151.2.176: {1865c5a2-f061-4256-aaff-e92277867544}}} }"} {"msg":"Profile for oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:40.233421474Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:40.233616966Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 55.37347ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:40.401877929Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 102.241067ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:40.470541612Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 212.668707ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:40.587941105Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 567.566986ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:40.813349833Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 541.398697ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:41.391631656Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 761.568912ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:41.94656836Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 693.342252ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:42.719753689Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 1.129316972s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:43.42289596Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\") failed: Property not found. retry in 1.100795747s","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:44.563707324Z","hostname":"sock","pid":1572,"zone":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.693144572Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1787"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.701487068Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1796"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.737105722Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1820"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.875747962Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:932"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-10-31T00:18:45.877147927Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.877323798Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.87748787Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-31T00:18:45.877147927Z }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.878035046Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1215"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.878047656Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-31T00:18:45.877147927Z }, migration_in: None, migration_out: None }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.878060516Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f130461b-c29f-4505-8150-5a38f9caa3ed","remote_addr":"[fd00:1122:3344:101::a]:57528","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7088882,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-10-31T00:18:45.890981608Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.890999009Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.891013749Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-31T00:18:45.890981608Z }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.891027259Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-31T00:18:45.890981608Z }, migration_in: None, migration_out: None }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:45.954511529Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ea621160-486a-40da-808c-37e37d264fdd","remote_addr":"[fd00:1122:3344:101::a]:57528","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":399,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.344766116Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50507"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.344894407Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"18a1be89-46ee-4fe1-8bad-51a108ceb0aa","remote_addr":"[fd00:1122:3344:101::c]:50507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.350879679Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36061"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.353444235Z","hostname":"sock","pid":1572,"uri":"/vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state","method":"GET","req_id":"4859292e-4689-44db-93fe-d7366a5003fe","remote_addr":"[fd00:1122:3344:101::c]:36061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.40674387Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {487b7899-43a4-486a-bd8d-ca0047fd7478: {10.151.2.176: {1865c5a2-f061-4256-aaff-e92277867544}, 10.151.2.175: {1865c5a2-f061-4256-aaff-e92277867544}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.406781571Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"83cd911a-cda0-4f68-8947-7eaae6831573","remote_addr":"[fd00:1122:3344:101::c]:50507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":149,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.424588283Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a1f6231b-b2f5-4324-935a-21e93daebf17","remote_addr":"[fd00:1122:3344:101::c]:50507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":234,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.645374664Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.645562116Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"71633de2-86eb-4b0d-927c-33aa5f27180d","remote_addr":"[fd00:1122:3344:101::c]:50493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":153,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.701403768Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57958"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.701497949Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.73767094Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.793695054Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"80b84384-0ab4-4e1b-ba41-7698a34b286c","remote_addr":"[fd00:1122:3344:101::c]:57958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92247,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:46.793926086Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"6be128d9-160b-4870-9b44-70e6df19deba","remote_addr":"[fd00:1122:3344:101::c]:57958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.162577192Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.162735273Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2598ff50-8aa9-4d52-b399-f84b46a34ea8","remote_addr":"[fd00:1122:3344:101::b]:36675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.218243702Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.220859669Z","hostname":"sock","pid":1572,"uri":"/vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state","method":"GET","req_id":"0340a8a2-148a-47af-8c98-c52e549de8ee","remote_addr":"[fd00:1122:3344:101::b]:35834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.244753093Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:50219"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.247063017Z","hostname":"sock","pid":1572,"uri":"/vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state","method":"GET","req_id":"bdb1558c-3a1c-4557-8ae6-a4ae753e92ea","remote_addr":"[fd00:1122:3344:101::a]:50219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.292965377Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {487b7899-43a4-486a-bd8d-ca0047fd7478: {10.151.2.176: {1865c5a2-f061-4256-aaff-e92277867544}, 10.151.2.175: {1865c5a2-f061-4256-aaff-e92277867544}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.293002288Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5e48216c-fc02-46c4-b3dd-113d77b8f093","remote_addr":"[fd00:1122:3344:101::b]:36675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":136,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.311210104Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d94853d5-3210-42dc-821d-a527e73b2816","remote_addr":"[fd00:1122:3344:101::b]:36675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":300,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.358174875Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59534"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.358440808Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"fbd5dc94-0d41-47a8-bea5-a92af51c404f","remote_addr":"[fd00:1122:3344:101::b]:59534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.359526479Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56100"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.35964386Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"74121ff0-2389-4eb4-ac34-66b42ac28812","remote_addr":"[fd00:1122:3344:101::a]:56100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.379764896Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59859"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.379887327Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.422727846Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.482569979Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ce1b0fb7-12a5-463f-97f3-3b2bf8de73e6","remote_addr":"[fd00:1122:3344:101::b]:59859","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102725,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.482777231Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {487b7899-43a4-486a-bd8d-ca0047fd7478: {10.151.2.176: {1865c5a2-f061-4256-aaff-e92277867544}, 10.151.2.175: {1865c5a2-f061-4256-aaff-e92277867544}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.482799741Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d0c46165-48de-4b68-a3ca-6d1cc88e399d","remote_addr":"[fd00:1122:3344:101::a]:56100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.482941043Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"fed7f214-32ce-46a5-b9dd-deaaa4936627","remote_addr":"[fd00:1122:3344:101::b]:59859","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.544768476Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53881"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.544963738Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"5736000e-5e12-444d-af46-9daaa89c5d88","remote_addr":"[fd00:1122:3344:101::a]:53881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.556629147Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6df17865-3619-4fe0-afb6-19c63703e558","remote_addr":"[fd00:1122:3344:101::a]:56100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":157,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.628467133Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55082"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.628567894Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.668723465Z","hostname":"sock","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.722812839Z","hostname":"sock","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a5e3b867-eb29-470c-b4a4-c9a06c47186b","remote_addr":"[fd00:1122:3344:101::a]:55082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94279,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:47.723071752Z","hostname":"sock","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0d96325e-9d70-4c68-a1e2-b93a92bfc817","remote_addr":"[fd00:1122:3344:101::a]:55082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.259825725Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49684"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.260617463Z","hostname":"sock","pid":1572,"uri":"/vmms/8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd/state","method":"PUT","req_id":"2b9ef96d-6a3d-487b-aea4-8dae7fa0434f","remote_addr":"[fd00:1122:3344:101::a]:49684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":662,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-10-31T00:18:56.260798915Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.260836456Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.260887866Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-31T00:18:56.260798915Z }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.260911566Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-31T00:18:56.260798915Z }, migration_in: None, migration_out: None }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-10-31T00:18:56.269651066Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.269669436Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.269687176Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-10-31T00:18:56.269651066Z }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.269695546Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:816","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.269714016Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:56.269778247Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"3209665516594333177"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:56.270366393Z","hostname":"sock","pid":1572,"component":"kstat-sampler-worker","sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"2300910021289903676"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.272565456Z","hostname":"sock","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.273327653Z","hostname":"sock","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/39b08bb4-95dc-4ffe-8006-7954c81e10b8/debug/bundle/zone\", \"/pool/int/3ddddc72-3f09-485d-a0b7-57e5a0e73bb2/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/1d6671a7-5622-4671-b7bf-0a757330cf6b/crypt/debug\", \"/pool/ext/7dd5e852-297b-4341-9eef-72d5112891d5/crypt/debug\", \"/pool/ext/efb62c8a-6541-476c-8447-df5266c7fcb3/crypt/debug\", \"/pool/ext/ca55523e-99d5-46dd-9043-66e4f721f4aa/crypt/debug\", \"/pool/ext/67e5554d-50d7-4b6f-b9c3-723d8231ab86/crypt/debug\", \"/pool/ext/1baca1d7-6a7c-4d5d-ade4-48f6b6e9c354/crypt/debug\", \"/pool/ext/a2173f6b-5437-4ef6-a016-ab4b5f89f5d8/crypt/debug\", \"/pool/ext/a536fa1e-3243-42d2-9446-aa31a45af0ee/crypt/debug\", \"/pool/ext/b43e5889-7a07-42ec-9599-254e55788daa/crypt/debug\"] }","zone_name":"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:56.89499831Z","hostname":"sock","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd\", bundle_id: 4d2b7470-6b2d-4389-a533-e45dc0315755 }, time_created: 2024-10-31T00:18:56.273540895Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","v":0,"name":"SledAgent","level":40,"time":"2024-10-31T00:18:56.895078881Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1032"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:58.623620975Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:58.640227865Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(15018464), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:58.640274845Z","hostname":"sock","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(15018464), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:58.641317566Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-31T00:18:58.641286306Z }, migration_in: None, migration_out: None }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:58.666893238Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-31T00:18:58.641286306Z }, migration_in: None, migration_out: None }","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:58.669647796Z","hostname":"sock","pid":1572,"instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","propolis_id":"8ad8cb2d-e5a6-4215-92bf-43e9cf757bdd","instance_id":"054c3697-a46c-4a3c-95ef-4a8d9a5537c2","component":"InstanceManager","file":"sled-agent/src/instance.rs:969"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.147849234Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62218"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.147966625Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"144fd918-0181-4b22-ad54-528c836beb02","remote_addr":"[fd00:1122:3344:101::a]:62218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.199133579Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.19917501Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.19918211Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"9d914043-02b6-4a97-911b-843da8bcffce","remote_addr":"[fd00:1122:3344:101::a]:62218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.223025454Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.223186496Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"388cf485-bf51-4f64-924e-4f3e91af3b8d","remote_addr":"[fd00:1122:3344:101::a]:34214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.232299229Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"40dcb312-b3ed-416f-88b9-af11b6b7416a","remote_addr":"[fd00:1122:3344:101::a]:62218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.695587624Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.695833676Z","hostname":"sock","pid":1572,"uri":"/v2p","method":"GET","req_id":"01b625d9-3143-4d81-95c4-59ad71071bdd","remote_addr":"[fd00:1122:3344:101::a]:51499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":179,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.823076029Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45699"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.823188331Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b466786c-273b-42ce-8305-a1b2ae27c8f4","remote_addr":"[fd00:1122:3344:101::a]:45699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":70,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.838244875Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.838265585Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"9e26a65d-32d4-487d-aa90-fb2311cb2886","remote_addr":"[fd00:1122:3344:101::a]:45699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:18:59.847381578Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"17092177-e219-40e9-bb94-876e043b3df9","remote_addr":"[fd00:1122:3344:101::a]:45699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":217,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:19:00.519139829Z","hostname":"sock","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61090"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:19:00.519252Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0b9a78ec-8d0e-4756-b7b4-e4aac67bbd05","remote_addr":"[fd00:1122:3344:101::a]:61090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":36,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:19:00.536673738Z","hostname":"sock","pid":1572,"sled_id":"f95bcc91-ac8f-446a-8a84-3ca7f7459fd7","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1189","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:19:00.536695648Z","hostname":"sock","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c6b75697-3f69-44dc-8dc1-c28eb2af9713","remote_addr":"[fd00:1122:3344:101::a]:61090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":43,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:19:00.545834442Z","hostname":"sock","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"31c22984-e404-4bcd-bf49-998bee81e852","remote_addr":"[fd00:1122:3344:101::a]:61090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":185,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:19:00.944651007Z","hostname":"sock","pid":1572,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-31T00:19:00.945225242Z","hostname":"sock","pid":1572,"uri":"/baseboard","method":"GET","req_id":"c17cea2d-9bb0-4e6a-9c07-6f9d5716d315","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62139","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"}